{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 波士顿房价预测案例——线性回归分析\n",
    "\n",
    "在这个案例中，我们将利用波士顿郊区的房屋信息数据训练和测试一个模型，并对模型的性能和预测能力进行测试。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1、导入必要的工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np  # 矩阵操作\n",
    "import pandas as pd # SQL数据处理\n",
    "\n",
    "from sklearn.metrics import r2_score  #评价回归预测模型的性能\n",
    "\n",
    "import matplotlib.pyplot as plt   #画图\n",
    "import seaborn as sns\n",
    "\n",
    "# 图形出现在Notebook里而不是新窗口\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. 读取数据\n",
    "做完特征工程后的数据，请先运行2_FE_BostonHousePrice.ipynb，得到文件FE_boston_housing.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>...</th>\n",
       "      <th>RAD_2</th>\n",
       "      <th>RAD_3</th>\n",
       "      <th>RAD_4</th>\n",
       "      <th>RAD_5</th>\n",
       "      <th>RAD_6</th>\n",
       "      <th>RAD_7</th>\n",
       "      <th>RAD_8</th>\n",
       "      <th>RAD_24</th>\n",
       "      <th>MEDV</th>\n",
       "      <th>log_MEDV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.419782</td>\n",
       "      <td>0.285654</td>\n",
       "      <td>-1.287909</td>\n",
       "      <td>-0.272599</td>\n",
       "      <td>-0.144217</td>\n",
       "      <td>0.413672</td>\n",
       "      <td>-0.120013</td>\n",
       "      <td>0.140214</td>\n",
       "      <td>-0.666608</td>\n",
       "      <td>-1.353192</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.159686</td>\n",
       "      <td>0.345176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.417339</td>\n",
       "      <td>-0.487292</td>\n",
       "      <td>-0.593381</td>\n",
       "      <td>-0.272599</td>\n",
       "      <td>-0.740262</td>\n",
       "      <td>0.194274</td>\n",
       "      <td>0.367166</td>\n",
       "      <td>0.557160</td>\n",
       "      <td>-0.987329</td>\n",
       "      <td>-0.475352</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.101524</td>\n",
       "      <td>0.084104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.417342</td>\n",
       "      <td>-0.487292</td>\n",
       "      <td>-0.593381</td>\n",
       "      <td>-0.272599</td>\n",
       "      <td>-0.740262</td>\n",
       "      <td>1.282714</td>\n",
       "      <td>-0.265812</td>\n",
       "      <td>0.557160</td>\n",
       "      <td>-0.987329</td>\n",
       "      <td>-0.475352</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.324247</td>\n",
       "      <td>1.266776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.416750</td>\n",
       "      <td>-0.487292</td>\n",
       "      <td>-1.306878</td>\n",
       "      <td>-0.272599</td>\n",
       "      <td>-0.835284</td>\n",
       "      <td>1.016303</td>\n",
       "      <td>-0.809889</td>\n",
       "      <td>1.077737</td>\n",
       "      <td>-1.106115</td>\n",
       "      <td>-0.036432</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.182758</td>\n",
       "      <td>1.170822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.412482</td>\n",
       "      <td>-0.487292</td>\n",
       "      <td>-1.306878</td>\n",
       "      <td>-0.272599</td>\n",
       "      <td>-0.835284</td>\n",
       "      <td>1.228577</td>\n",
       "      <td>-0.511180</td>\n",
       "      <td>1.077737</td>\n",
       "      <td>-1.106115</td>\n",
       "      <td>-0.036432</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.487503</td>\n",
       "      <td>1.373242</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       CRIM        ZN     INDUS      CHAS       NOX        RM       AGE  \\\n",
       "0 -0.419782  0.285654 -1.287909 -0.272599 -0.144217  0.413672 -0.120013   \n",
       "1 -0.417339 -0.487292 -0.593381 -0.272599 -0.740262  0.194274  0.367166   \n",
       "2 -0.417342 -0.487292 -0.593381 -0.272599 -0.740262  1.282714 -0.265812   \n",
       "3 -0.416750 -0.487292 -1.306878 -0.272599 -0.835284  1.016303 -0.809889   \n",
       "4 -0.412482 -0.487292 -1.306878 -0.272599 -0.835284  1.228577 -0.511180   \n",
       "\n",
       "        DIS       TAX   PTRATIO    ...     RAD_2  RAD_3  RAD_4  RAD_5  RAD_6  \\\n",
       "0  0.140214 -0.666608 -1.353192    ...         0      0      0      0      0   \n",
       "1  0.557160 -0.987329 -0.475352    ...         1      0      0      0      0   \n",
       "2  0.557160 -0.987329 -0.475352    ...         1      0      0      0      0   \n",
       "3  1.077737 -1.106115 -0.036432    ...         0      1      0      0      0   \n",
       "4  1.077737 -1.106115 -0.036432    ...         0      1      0      0      0   \n",
       "\n",
       "   RAD_7  RAD_8  RAD_24      MEDV  log_MEDV  \n",
       "0      0      0       0  0.159686  0.345176  \n",
       "1      0      0       0 -0.101524  0.084104  \n",
       "2      0      0       0  1.324247  1.266776  \n",
       "3      0      0       0  1.182758  1.170822  \n",
       "4      0      0       0  1.487503  1.373242  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# path to where the data lies\n",
    "#dpath = './data/'\n",
    "df = pd.read_csv(\"FE_boston_housing.csv\")\n",
    "\n",
    "#通过观察前5行，了解数据每列（特征）的概况\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  数据基本信息\n",
    "样本数目、特征维数\n",
    "每个特征的类型、空值样本的数目、数据类型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 从原始数据中分离输入特征x和输出y\n",
    "y = df[\"MEDV\"]\n",
    "\n",
    "X = df.drop([\"MEDV\", \"log_MEDV\"], axis = 1)\n",
    "\n",
    "#特征名称，用于后续显示权重系数对应的特征\n",
    "feat_names = X.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "当数据量比较大时，可用train_test_split从训练集中分出一部分做校验集；\n",
    "样本数目较少时，建议用交叉验证。\n",
    "在线性回归中，留一交叉验证有简便计算方式。\n",
    "\n",
    "下面将训练数据分割成训练集和测试集，只是让大家对模型的训练误差、校验集上的测试误差估计、和测试集上的测试误差做个比较。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(404, 21)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#将数据分割训练数据与测试数据\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 随机采样20%的数据构建测试样本，其余作为训练样本\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=33, test_size=0.2)\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3、确定模型类型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 尝试缺省参数的线性回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>coef</th>\n",
       "      <th>columns</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.505543</td>\n",
       "      <td>RAD_24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.297984</td>\n",
       "      <td>RM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.184468</td>\n",
       "      <td>RAD_7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.147403</td>\n",
       "      <td>ZN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.147279</td>\n",
       "      <td>RAD_8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.134516</td>\n",
       "      <td>RAD_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.088869</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.074143</td>\n",
       "      <td>CHAS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.017002</td>\n",
       "      <td>INDUS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>-0.001742</td>\n",
       "      <td>AGE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>-0.042599</td>\n",
       "      <td>RAD_4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>-0.056407</td>\n",
       "      <td>RAD_5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.104955</td>\n",
       "      <td>CRIM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.176803</td>\n",
       "      <td>NOX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-0.178692</td>\n",
       "      <td>TAX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>-0.200660</td>\n",
       "      <td>RAD_2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-0.209666</td>\n",
       "      <td>PTRATIO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>-0.272976</td>\n",
       "      <td>RAD_6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-0.361874</td>\n",
       "      <td>DIS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>-0.399165</td>\n",
       "      <td>RAD_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>-0.459577</td>\n",
       "      <td>LSTAT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        coef  columns\n",
       "20  0.505543   RAD_24\n",
       "5   0.297984       RM\n",
       "18  0.184468    RAD_7\n",
       "1   0.147403       ZN\n",
       "19  0.147279    RAD_8\n",
       "14  0.134516    RAD_3\n",
       "10  0.088869        B\n",
       "3   0.074143     CHAS\n",
       "2   0.017002    INDUS\n",
       "6  -0.001742      AGE\n",
       "15 -0.042599    RAD_4\n",
       "16 -0.056407    RAD_5\n",
       "0  -0.104955     CRIM\n",
       "4  -0.176803      NOX\n",
       "8  -0.178692      TAX\n",
       "13 -0.200660    RAD_2\n",
       "9  -0.209666  PTRATIO\n",
       "17 -0.272976    RAD_6\n",
       "7  -0.361874      DIS\n",
       "12 -0.399165    RAD_1\n",
       "11 -0.459577    LSTAT"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 线性回归\n",
    "#class sklearn.linear_model.LinearRegression(fit_intercept=True, normalize=False, copy_X=True, n_jobs=1)\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "# 1.使用默认配置初始化学习器实例\n",
    "lr = LinearRegression()\n",
    "\n",
    "# 2.用训练数据训练模型参数\n",
    "lr.fit(X_train, y_train)\n",
    "\n",
    "# 3. 用训练好的模型对测试集进行预测\n",
    "y_test_pred_lr = lr.predict(X_test)\n",
    "y_train_pred_lr = lr.predict(X_train)\n",
    "\n",
    "\n",
    "# 看看各特征的权重系数，系数的绝对值大小可视为该特征的重要性\n",
    "fs = pd.DataFrame({\"columns\":list(feat_names), \"coef\":list((lr.coef_.T))})\n",
    "fs.sort_values(by=['coef'],ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.1.1 模型评价"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The r2 score of LinearRegression on test is 0.693978981051\n",
      "The r2 score of LinearRegression on train is 0.754914643687\n"
     ]
    }
   ],
   "source": [
    "# 使用r2_score评价模型在测试集和训练集上的性能，并输出评估结果\n",
    "#测试集\n",
    "print 'The r2 score of LinearRegression on test is', r2_score(y_test, y_test_pred_lr)\n",
    "#训练集\n",
    "print 'The r2 score of LinearRegression on train is', r2_score(y_train, y_train_pred_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAekAAAFsCAYAAADos0H8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAG+tJREFUeJzt3X+8VXWd7/HXRzhC41EIBUXQwNLy\nJ2BHgkyjqHTCxHlMpV1/4EPU0pqH1e2XdcszTt3y6rXJq3XH6YealqhT4dT4GLsEQ3pLB4z8RQWZ\n3EBCpBSxTH587h97QQc4h7PPL/b3cF7Px2M/zt5rrb3W57vWgff5rvXda0dmIkmSyrNXowuQJEnt\nM6QlSSqUIS1JUqEMaUmSCmVIS5JUKENakqRCGdIa8CLisYiY1ug6Giki/iYifhsRGyJi0m7c7oaI\nOKyDeedHxH29tJ0nI+ItvbEuaXcypLVHa+8/5x3/88/MozNzQSfrGRcRGRGD+6jURrsG+EBmNmfm\nz3acWbX9hSpUV0XEtRExqKcbrbb3RE/XI+2pDGmpAAWE/yuAxzpZZkJmNgNvBM4ELujzqqQBzpDW\ngNe2tx0RkyNiUUSsj4g1EXFttdjC6uezVW9yakTsFRH/LSJWRMTTEXFLRAxrs97zqnnrIuLTO2yn\nNSLuiohbI2I9cH617Z9ExLMRsToiro+IvdusLyPi0ohYFhHPR8Q/RMQrq/esj4g72i6/QxvbrTUi\nhkTEBmAQ8POI+HVn+yszlwP3AxPbrH9YRHytqntVRHx2a087Il4VEf8REc9FxDMRMWeHNr2qer5/\nRNxdteVB4JVtltvpTEZELIiIC6vnr4yIH1X7+pmIuC0ihnewLzo6xlJxDGlpe18CvpSZ+1ELiTuq\n6SdXP4dXp2h/ApxfPd4EHAY0A9cDRMRRwJeBs4HRwDBgzA7bmgncBQwHbgM2Ax8CDgCmAtOBS3d4\nz6nAa4EpwMeAG6ttHAIcA7yng3a1W2tm/rnqHUOtp/zK9t/+FxHxGuAkYHmbyTcDm4BXAZOAtwEX\nVvP+AbgXeDkwFvhfHaz6BuBFavvrArrWUw/g88DBwJHU9kdrB8t2dIyl4hjSGgi+V/VOn42IZ6mF\nZ0c2Aq+KiAMyc0Nm/nQXy54NXJuZT2TmBuBy4Kyqt/dO4F8z877MfAn4DLDjjfJ/kpnfy8wtmfmn\nzFycmT/NzE2Z+STwT9ROLbd1VWauz8zHgEeBe6vtPwfcQy0gu1prvR6KiBeApcACqv0YEQcCfw18\nMDNfyMyngS8CZ1Xv20jtdPrBmfliZu40GKzqdf8t8JlqHY9SC/66ZObyzPxh9UfHWuBadt53W3Xl\nGEsNZUhrIDgjM4dvfbBz77St2cARwC8i4j8j4rRdLHswsKLN6xXAYODAat5vt87IzD8C63Z4/2/b\nvoiIIyLi+xHxu+oU+H+n1qtua02b539q53Uz7dtVrfU6vlr/mcDrgH2q6a8AmoDVbf4Q+idgVDX/\nY9R6ug9GbSR9ez3kkVU9bffJinaWa1dEjIqI26tT7euBW9l5323VlWMsNZQhLbWRmcsy8z3UAuYq\n4K6I2Iede8EAT1ELqK0OpXbKdw2wmtqpXQAi4mXA/jtubofXXwF+ARxenYr9JLVw6w27qrVuWXMH\n8BNqZwegFqx/Bg5o88fQfpl5dPWe32XmRZl5MPBe4Mtbr0O3sbaq55AdatzqhernX7WZdlCb55+n\ntj+Pq/bdOXSw73ZxjKXiGNJSGxFxTkSMzMwtwLPV5M3UQmQLteu5W30b+FBEjI+IZmo93zmZuYna\nteZ3RMTrq8Fcf0/ngbsvsB7YUF33vaTXGrbrWrvjC8DFEXFQZq6mds35f0bEftUgtVdGxBsBIuJd\nEbH1D5Y/UAvTzW1Xlpmbge8ArRHxV9U1/Vlt5q8FVgHnRMSgqjfe9vr5vsAGagP7xgAf7ajwXRxj\nqTiGtLS9U4HHqhHPXwLOqq6j/hH4HHB/dUp3CvB14JvURn7/htqgp78DqK4Z/x1wO7Ve9fPA09R6\nnB35CPBfqmX/GZizi2W7qsNauyMzHwH+g7+E4XnA3sDj1IL4LmoDwABOAB6o9undwGWZ+Zt2VvsB\naqfTfwfcBHxjh/kXVdtbBxwN/N828/6e2un454AfUAv8jrR7jHfdYqkxIrO9s3iSelPVe32W2qns\n9gJKknZiT1rqIxHxjurU7T7U7uj1CPBkY6uS1J8Y0lLfmUltwNZTwOHUTqt66kpS3TzdLUlSoexJ\nS5JUqN16U/8DDjggx40btzs3KUlSURYvXvxMZo6sZ9ndGtLjxo1j0aJFu3OTkiQVJSLqvpuep7sl\nSSqUIS1JUqEMaUmSCrVbr0lLktq3ceNGVq5cyYsveofSPcXQoUMZO3YsTU1N3V6HIS1JBVi5ciX7\n7rsv48aNI6K3vvxMjZKZrFu3jpUrVzJ+/Phur8fT3ZJUgBdffJH999/fgN5DRAT7779/j8+MGNKS\nVAgDes/SG8fTkJYkqVBek5akArW27v71DRo0iGOPPZZNmzYxfvx4vvnNbzJ8+PAub+vCCy/kwx/+\nMEcdddR202+66SYWLVrE9ddf3+V1AjQ3N7Nhw4a6lp02bRrXXHMNLS0t26YtWrSIW265heuuu65b\n228Ee9KSJABe9rKXsWTJEh599FFGjBjBDTfc0K31fPWrX90poEvQ0tLS5wG9efPmXl2fIS1J2snU\nqVNZtWrVttdXX301J5xwAscddxxXXHEFAC+88AIzZsxgwoQJHHPMMcyZMweo9WK33gL6G9/4Bkcc\ncQRvfOMbuf/++7et7/zzz+euu+7a9rq5uRmADRs2MH36dI4//niOPfZY5s6du1Ntq1ev5uSTT2bi\nxIkcc8wx/PjHP66rTQsWLOC0004DoLW1lQsuuIBp06Zx2GGHbRfet956K5MnT2bixIm8973v3Ra8\nl1xyCS0tLRx99NHb9gHUbnl95ZVX8oY3vIE777yzrlrq5eluSdJ2Nm/ezLx585g9ezYA9957L8uW\nLePBBx8kMzn99NNZuHAha9eu5eCDD+YHP/gBAM8999x261m9ejVXXHEFixcvZtiwYbzpTW9i0qRJ\nu9z20KFD+e53v8t+++3HM888w5QpUzj99NO3G4T1rW99i1NOOYVPfepTbN68mT/+8Y/daucvfvEL\n5s+fz/PPP8+rX/1qLrnkEpYvX86cOXO4//77aWpq4tJLL+W2227jvPPO43Of+xwjRoxg8+bNTJ8+\nnYcffpjjjjtuW9333Xdft+rYFUNakgTAn/70JyZOnMiTTz7Ja1/7Wt761rcCtZC+9957twXshg0b\nWLZsGSeddBIf+chH+PjHP85pp53GSSedtN36HnjgAaZNm8bIkbUvfDrzzDP51a9+tcsaMpNPfvKT\nLFy4kL322otVq1axZs0aDjrooG3LnHDCCVxwwQVs3LiRM844g4kTJ3arvTNmzGDIkCEMGTKEUaNG\nsWbNGubNm8fixYs54YQTtu2TUaNGAXDHHXdw4403smnTJlavXs3jjz++LaTPPPPMbtXQGU93S5KA\nv1yTXrFiBS+99NK2a9KZyeWXX86SJUtYsmQJy5cvZ/bs2RxxxBEsXryYY489lssvv5wrr7xyp3V2\n9DGkwYMHs2XLlm3rf+mllwC47bbbWLt2LYsXL2bJkiUceOCBO33W+OSTT2bhwoWMGTOGc889l1tu\nuaVb7R0yZMi254MGDWLTpk1kJrNmzdrW1l/+8pe0trbym9/8hmuuuYZ58+bx8MMPM2PGjO3q2mef\nfbpVQ2fsSatL6h1x2tsjUyXtPsOGDeO6665j5syZXHLJJZxyyil8+tOf5uyzz6a5uZlVq1bR1NTE\npk2bGDFiBOeccw7Nzc3cdNNN263nda97HZdddhnr1q1jv/32484772TChAlA7Tru4sWLefe7383c\nuXPZuHEjUDtlPmrUKJqampg/fz4rVuz8rY4rVqxgzJgxXHTRRbzwwgs89NBDnHfeeb3S9unTpzNz\n5kw+9KEPMWrUKH7/+9/z/PPPs379evbZZx+GDRvGmjVruOeee5g2bVqvbHNXDGlJKlCj/9CdNGkS\nEyZM4Pbbb+fcc89l6dKlTJ06FagN8rr11ltZvnw5H/3oR9lrr71oamriK1/5ynbrGD16NK2trUyd\nOpXRo0dz/PHHbxuEddFFFzFz5kwmT57M9OnTt/VEzz77bN7xjnfQ0tLCxIkTec1rXrNTbQsWLODq\nq6+mqamJ5ubmDnvSM2bM2Hbf7KlTp/L+97+/03YfddRRfPazn+Vtb3sbW7ZsoampiRtuuIEpU6Yw\nadIkjj76aA477DBOPPHE+ndmD0Rm7pYNAbS0tOTWEX/qn+xJS31j6dKlHHnkkY0uQ72sveMaEYsz\ns6WDt2zHa9KSJBXKkJYkqVCGtCQVYndeflTf643j6cAx9RteD9eebOjQoaxbt86vq9xDbP0+6aFD\nh/ZoPYa0JBVg7NixrFy5krVr1za6FPWSoUOHMnbs2B6tw5CWpAI0NTUxfvz4RpehwnhNWpKkQhnS\nkiQVypCWJKlQXpNWQzkSW5I6Zk9akqRCGdKSJBXKkJYkqVCGtCRJhTKkJUkqlCEtSVKhDGlJkgpl\nSEuSVChDWpKkQhnSkiQVypCWJKlQhrQkSYUypCVJKpQhLUlSoQxpSZIKZUhLklQoQ1qSpEIZ0pIk\nFcqQliSpUIa0JEmFMqQlSSqUIS1JUqEMaUmSCmVIS5JUKENakqRCGdKSJBXKkJYkqVCGtCRJhao7\npCNiUET8LCK+X70eHxEPRMSyiJgTEXv3XZmSJA08XelJXwYsbfP6KuCLmXk48Adgdm8WJknSQFdX\nSEfEWGAG8NXqdQBvBu6qFrkZOKMvCpQkaaCqtyf9j8DHgC3V6/2BZzNzU/V6JTCml2uTJGlA6zSk\nI+I04OnMXNx2cjuLZgfvvzgiFkXEorVr13azTEmSBp56etInAqdHxJPA7dROc/8jMDwiBlfLjAWe\nau/NmXljZrZkZsvIkSN7oWRJkgaGTkM6My/PzLGZOQ44C/hRZp4NzAfeWS02C5jbZ1VKkjQA9eRz\n0h8HPhwRy6ldo/5a75QkSZIABne+yF9k5gJgQfX8CWBy75ckSZLAO45JklQsQ1qSpEIZ0pIkFcqQ\nliSpUIa0JEmFMqQlSSqUIS1JUqG69DlpqV6trY2uQJL6P3vSkiQVypCWJKlQhrQkSYUypCVJKpQh\nLUlSoQxpSZIKZUhLklQoQ1qSpEIZ0pIkFcqQliSpUIa0JEmFMqQlSSqUIS1JUqEMaUmSCmVIS5JU\nKENakqRCGdKSJBXKkJYkqVCGtCRJhTKkJUkqlCEtSVKhDGlJkgplSEuSVChDWpKkQhnSkiQVypCW\nJKlQhrQkSYUypCVJKpQhLUlSoQxpSZIKZUhLklQoQ1qSpEIZ0pIkFcqQliSpUIa0JEmFMqQlSSqU\nIS1JUqEMaUmSCmVIS5JUKENakqRCGdKSJBXKkJYkqVCGtCRJhTKkJUkq1OBGF6DGa21tdAWSpPbY\nk5YkqVCGtCRJhTKkJUkqVKchHRFDI+LBiPh5RDwWEX9fTR8fEQ9ExLKImBMRe/d9uZIkDRz19KT/\nDLw5MycAE4FTI2IKcBXwxcw8HPgDMLvvypQkaeDpNKSzZkP1sql6JPBm4K5q+s3AGX1SoSRJA1Rd\nH8GKiEHAYuBVwA3Ar4FnM3NTtchKYEwH770YuBjg0EMP7Wm9Uqfq/UiZHz2TVLq6Bo5l5ubMnAiM\nBSYDR7a3WAfvvTEzWzKzZeTIkd2vVJKkAaZLo7sz81lgATAFGB4RW3viY4Gnerc0SZIGtnpGd4+M\niOHV85cBbwGWAvOBd1aLzQLm9lWRkiQNRPVckx4N3Fxdl94LuCMzvx8RjwO3R8RngZ8BX+vDOiVJ\nGnA6DenMfBiY1M70J6hdn5YkSX3AO45JklQoQ1qSpEIZ0pIkFcqQliSpUIa0JEmFMqQlSSqUIS1J\nUqEMaUmSCmVIS5JUKENakqRCGdKSJBXKkJYkqVCGtCRJhTKkJUkqlCEtSVKhDGlJkgplSEuSVChD\nWpKkQhnSkiQVypCWJKlQhrQkSYUypCVJKpQhLUlSoQxpSZIKZUhLklQoQ1qSpEINbnQBUqO0tvbu\ncpLU2+xJS5JUKENakqRCGdKSJBXKkJYkqVCGtCRJhTKkJUkqlCEtSVKhDGlJkgplSEuSVChDWpKk\nQhnSkiQVypCWJKlQhrQkSYUypCVJKpQhLUlSoQxpSZIKZUhLklQoQ1qSpEIZ0pIkFcqQliSpUIa0\nJEmFMqQlSSqUIS1JUqEMaUmSCmVIS5JUqMGNLkB9p7W10RVIknrCnrQkSYUypCVJKlSnIR0Rh0TE\n/IhYGhGPRcRl1fQREfHDiFhW/Xx535crSdLAUU9PehPwXzPzSGAK8P6IOAr4BDAvMw8H5lWvJUlS\nL+k0pDNzdWY+VD1/HlgKjAFmAjdXi90MnNFXRUqSNBB16Zp0RIwDJgEPAAdm5mqoBTkwqoP3XBwR\niyJi0dq1a3tWrSRJA0jdIR0RzcC/AB/MzPX1vi8zb8zMlsxsGTlyZHdqlCRpQKorpCOiiVpA35aZ\n36kmr4mI0dX80cDTfVOiJEkDUz2juwP4GrA0M69tM+tuYFb1fBYwt/fLkyRp4KrnjmMnAucCj0TE\nkmraJ4EvAHdExGzg/wHv6psSJUkamDoN6cy8D4gOZk/v3XIkSdJW3nFMkqRCGdKSJBXKkJYkqVCG\ntCRJhTKkJUkqlCEtSVKhDGlJkgpVz81MVJjW1kZXIEnaHexJS5JUKENakqRCGdKSJBXKkJYkqVCG\ntCRJhTKkJUkqlCEtSVKhDGlJkgplSEuSVChDWpKkQhnSkiQVypCWJKlQfsFGQfzijDJ15bh4DCX1\nJnvSkiQVypCWJKlQhrQkSYUypCVJKpQhLUlSoQxpSZIKZUhLklQoQ1qSpEIZ0pIkFcqQliSpUIa0\nJEmFMqQlSSqUX7Ah9aJ6v2DDL+KQVA970pIkFcqQliSpUIa0JEmFMqQlSSqUIS1JUqEMaUmSCmVI\nS5JUKENakqRCGdKSJBXKkJYkqVCGtCRJhTKkJUkqlCEtSVKhDGlJkgplSEuSVChDWpKkQhnSkiQV\nypCWJKlQhrQkSYUypCVJKpQhLUlSoToN6Yj4ekQ8HRGPtpk2IiJ+GBHLqp8v79syJUkaeOrpSd8E\nnLrDtE8A8zLzcGBe9VqSJPWiTkM6MxcCv99h8kzg5ur5zcAZvVyXJEkD3uBuvu/AzFwNkJmrI2JU\nRwtGxMXAxQCHHnpoNzfXf7W2NroCSVJ/1ecDxzLzxsxsycyWkSNH9vXmJEnaY3Q3pNdExGiA6ufT\nvVeSJEmC7of03cCs6vksYG7vlCNJkraq5yNY3wZ+Arw6IlZGxGzgC8BbI2IZ8NbqtSRJ6kWdDhzL\nzPd0MGt6L9ciSZLa8I5jkiQVqrsfwZK0G9T7ET4/6iftmexJS5JUKENakqRCGdKSJBXKkJYkqVCG\ntCRJhTKkJUkqlCEtSVKhDGlJkgplSEuSVChDWpKkQhnSkiQVypCWJKlQhrQkSYUypCVJKpQhLUlS\noQxpSZIKZUhLklQoQ1qSpEIZ0pIkFcqQliSpUIa0JEmFMqQlSSqUIS1JUqEMaUmSCmVIS5JUqMGN\nLkAaiFpbG13BnqHe/ej+Vn9lT1qSpEIZ0pIkFcqQliSpUIa0JEmFMqQlSSqUo7u7ydGiKklXfh/9\n3ZX6D3vSkiQVypCWJKlQhrQkSYUypCVJKpQhLUlSoQxpSZIK5UewJPVIX3zJhR8Tk2rsSUuSVChD\nWpKkQhnSkiQVypCWJKlQhrQkSYVydLc0wDRq5HR/GLHtCPSBpS8+mdDb7ElLklQoQ1qSpEIZ0pIk\nFcqQliSpUIa0JEmF6tejux2JKWlP0Rf/R+1J/+/1h5HYfcGetCRJhTKkJUkqlCEtSVKhehTSEXFq\nRPwyIpZHxCd6qyhJktSDkI6IQcANwF8DRwHviYijeqswSZIGup70pCcDyzPzicx8CbgdmNk7ZUmS\npMjM7r0x4p3AqZl5YfX6XOB1mfmBHZa7GLi4evlq4JfdL7dPHQA80+giepHtKd+e1ibbUzbbU45X\nZObIehbsyeeko51pOyV+Zt4I3NiD7ewWEbEoM1saXUdvsT3l29PaZHvKZnv6p56c7l4JHNLm9Vjg\nqZ6VI0mStupJSP8ncHhEjI+IvYGzgLt7pyxJktTt092ZuSkiPgD8OzAI+HpmPtZrle1+xZ+S7yLb\nU749rU22p2y2px/q9sAxSZLUt7zjmCRJhTKkJUkq1IAN6Yh4V0Q8FhFbIqLDYfz95danETEiIn4Y\nEcuqny/vYLnNEbGkehQ30K+z/R0RQyJiTjX/gYgYt/urrF8d7Tk/Ita2OSYXNqLOekXE1yPi6Yh4\ntIP5ERHXVe19OCKO3901dkUd7ZkWEc+1OT6f2d01dkVEHBIR8yNiafX/22XtLNNvjlGd7elXx6jL\nMnNAPoAjqd1cZQHQ0sEyg4BfA4cBewM/B45qdO0d1Po/gE9Uzz8BXNXBchsaXesu2tDp/gYuBf53\n9fwsYE6j6+5he84Hrm90rV1o08nA8cCjHcx/O3APtfsoTAEeaHTNPWzPNOD7ja6zC+0ZDRxfPd8X\n+FU7v3P95hjV2Z5+dYy6+hiwPenMXJqZnd39rD/d+nQmcHP1/GbgjAbW0l317O+27bwLmB4R7d1Y\npwT96fenLpm5EPj9LhaZCdySNT8FhkfE6N1TXdfV0Z5+JTNXZ+ZD1fPngaXAmB0W6zfHqM727NEG\nbEjXaQzw2zavV1LuL8iBmbkaar/YwKgOlhsaEYsi4qcRUVqQ17O/ty2TmZuA54D9d0t1XVfv78/f\nVqcd74qIQ9qZ35/0p38z9ZoaET+PiHsi4uhGF1Ov6lLQJOCBHWb1y2O0i/ZAPz1G9ejJbUGLFxH/\nBzionVmfysy59ayinWkN+8zartrThdUcmplPRcRhwI8i4pHM/HXvVNhj9ezvoo5JJ+qp9V+Bb2fm\nnyPifdTOEry5zyvrO/3p+NTjIWr3Wd4QEW8Hvgcc3uCaOhURzcC/AB/MzPU7zm7nLUUfo07a0y+P\nUb326JDOzLf0cBVF3fp0V+2JiDURMTozV1enrp7uYB1PVT+fiIgF1P4yLSWk69nfW5dZGRGDgWGU\ne7qy0/Zk5ro2L/8ZuGo31NWXivo301NtAyEz/y0ivhwRB2RmsV/sEBFN1ALttsz8TjuL9Ktj1Fl7\n+uMx6gpPd+9af7r16d3ArOr5LGCnMwUR8fKIGFI9PwA4EXh8t1XYuXr2d9t2vhP4UVajRwrUaXt2\nuBZ4OrVrbv3Z3cB51QjiKcBzWy/D9EcRcdDWMQ8RMZna/5nrdv2uxqlq/RqwNDOv7WCxfnOM6mlP\nfztGXdbokWuNegB/Q+0vyj8Da4B/r6YfDPxbm+XeTm1E4a+pnSZveO0dtGd/YB6wrPo5opreAny1\nev564BFqo4wfAWY3uu522rHT/gauBE6vng8F7gSWAw8ChzW65h625/PAY9UxmQ+8ptE1d9KebwOr\ngY3Vv5/ZwPuA91XzA7ihau8jdPDJiVIedbTnA22Oz0+B1ze65k7a8wZqp64fBpZUj7f312NUZ3v6\n1THq6sPbgkqSVChPd0uSVChDWpKkQhnSkiQVypCWJKlQhrQkSYUypCVJKpQhLUlSof4/4aA7PE7A\n4SQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a1369dc50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#在训练集上观察预测残差的分布，看是否符合模型假设：噪声为0均值的高斯噪声\n",
    "f, ax = plt.subplots(figsize=(7, 5)) \n",
    "f.tight_layout() \n",
    "ax.hist(y_train - y_train_pred_lr, bins=40, label='Residuals Linear', color='b', alpha=.5); \n",
    "ax.set_title(\"Histogram of Residuals\") \n",
    "ax.legend(loc='best');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "残差分布和高斯分布比较匹配，但还是大值这边的尾巴更长"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARgAAADQCAYAAADcQn7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJztnXl81OW1/98nwySZhCVhUTCAuGIR\nhBSKKC5FUVHR4kKlVUvd77X+LFVpQa3Lba1U3Lm9rVp3kCtWRQUFd1yugLEsKkhdKmhAZEsQSEgy\nc35/zAxOklm+yXxnvlnO+/XKi3xnvvM8Jwnzmec55zzniKpiGIaRCXK8NsAwjLaLCYxhGBnDBMYw\njIxhAmMYRsYwgTEMI2OYwBiGkTFMYAzDyBgmMIZhZAwTGMMwMkYHrw1oCt27d9d+/fp5bYZhtHs+\n+OCDzaraI9V9rUpg+vXrR1lZmddmGEa7R0TWOrnPtkiGYWQMExjDMDKGZwIjIvkislREVojIxyJy\ns1e2GIaRGbz0wewGjlPVHSLiB94RkZdUdbGHNhmG4SKerWA0zI7IpT/yZcVpDMMDKisrWb16tevj\neuqDERGfiCwHvgVeUdUlce65VETKRKRs06ZN2TfSMNo4ixYt4rDDDuOMM84gGAy6OranAqOqQVUd\nAvQGhovIwDj33K+qw1R1WI8eKcPuhmE4ZPfu3UyePJlRo0aRm5vLI488gs/nc3WOFhFFUtUK4E1g\njMemGEa7YdKkSdx+++1cdtllLF++nBEjRrg+h5dRpB4iUhT5PgCMBj7xyh7DaA8Eg0EqKysBuPba\na5k/fz5//etfKSwszMh8XkaRegGPioiPsNDNUdV5HtpjGG2atWvXMnHiRPLy8liwYAF9+vShT58+\nGZ3TM4FR1ZVAqVfzG0Z7QVV5/PHHueKKKwCYMWNG1uZuET4YwzAyw9atWxk/fjwTJ05kyJAhrFy5\nkokTJyIiWZnfBMYw2jAiwrJly7jtttt44403yHY1glZ1mtowjNTs3LmTe+65h2uuuYbi4mJWrVpF\nXl6eJ7bYCsYw2hBLliyhtLSU66+/ntdeew3AM3EBExjDaBPU1tZy0003MXLkSKqrq3n99dc5+eST\nvTbLtkiG0Ra4+OKLeeyxxzj//PO59957KSoq8tokwATGMFotqkpNTQ15eXlcddVVjB07lvHjx3tt\nVj1MYAyjFbJhwwYuvPBC9tlnHx588EEGDx7M4MGDmzXW3GXlTF+4hvUVVexTFGDySf0ZV1riip3m\ngzGMVsbTTz/NwIEDWbRoEUOHDkW1+VVO5i4rZ+ozH1JeUYUC5RVVTH3mQ+YuK3fFVhMYw2glVFZW\nMnHiRM4++2wOOOAAli1bxuWXX55W0tz0hWuoqq1foqGqNsj0hWvSNRewLZJhtBoqKyuZN28eN9xw\nA9dffz1+v7/JYzTcDpVXVMW9L9HjTcUExjBaMNXV1Tz++ONcfPHF9O3bly+++IIuXbo0a6zodii6\nYkkmIj6XjhLYFskwWigrV65k+PDhXHrppbz99tsAzRYXiL8dSkQwDb9OLCYwhtHCCAaDTJ8+nR/9\n6Eds2rSJF198kWOOOSbtcdc3YdtTUhRIez6wLZJhtDjOP/98Zs+ezZlnnsl9991H9+7dXRk3mc8l\nlhyBySf1d2VOLyva9RGRN0RkdaQv0q+9ssUwvEZV9xTcvvDCC3nkkUf4xz/+4Zq4QFg0Av7UNXdD\nCmVrt7oyp5dbpDrgalX9ATAC+JWIDPDQHsPwhE2bNnHWWWdx883h3oOjR4+OW7Nl7rJyRk57nf2m\nzGfktNebnKsyrrSEs4aW4MR9O2vJuiaNnQgvK9ptADZEvv9ORFYDJcAqr2wyjGwzf/58LrroIrZt\n28ZRRx2V8L54EaCpz3wIhIXDaTbuG59sctR8zCUfb8vwwYhIP8LlM+P2RQIuBejbt29W7TKMTLFz\n506uueYa/va3vzFo0CBefvllDjvssIT3p0qISyY+sTTF0esGnkeRRKQj8DQwSVW3N3ze+iIZbZHP\nPvuMhx9+mGuuuYalS5cmFRdILAzlFVVcPWeF42zcfVyKDjnF686OfsLiMktVn/HSFsPINLW1tTz3\n3HMADB48mM8//5zp06eTn5+f1L8yd1k5OUkS3xLlrMQTJaeOXrfwbIskYQ/Wg8BqVb3TKzsMwy2S\n+UHWrFnDeeedR1lZGWVlZQwdOpSSkpI9r0u0xYHw9qc5iW/xVitRe6J25ojEHbst5MGMBM4HPoz0\npwa4VlVf9NAmw2gWiURCVVn/3nNMnjyZQCDAU089xdChQ+u9NpV/xWn2bSwBv69RLktDAbzrnCEA\nTH5qBbWh70XGnyOu5cF4GUV6BxxFzAzDc1JFaRKJxCW/PI8tHy5izJgxPPTQQ/Tq1avR2In8K811\nyPpEuPXMQfXsSySAZw0tabSCceuYALQAJ69htHSc1ExpKAbRGi2y7zD+8pe/8OKLL8YVF0jseN2n\nKJDwuWSfzCHVRtGjRAI4a8k6Qg30JKRw0/MfJ5nBOSYwhhEhkaP1puc/ThmliQpBqHoHm+fdwY4V\nCwA4eOQpe2q2JBo/keN15+46Rh3So9FzAb+Pc0f0TXjiOZ4oJVoNJVqsVFTVxn+iibSIPBjD8JpE\nW4iytVsTvtli37STT+rPlXfOZP1zdxDcsQV/933r+UFSJcoB3PzCx2zb9f1cFVW1zFq8DiW87Qmq\nUhKzPRu2b9d6Y0J83ws4P4fkNraCMQwSbyFmL/kq4WuiK4Xq6mrennknX82cQofcXHqeN50uI84m\nr8P3b69UjtxxpSUU5Db+vI8uMIKqe8QjKkjjSku49cxBlBQFEMKRn1vPHATQaKUUb5WUjXC1CYxh\nkLj4UjKHZ3Sl8N5773HnnXcyZvxE9r3ov8nbJ/x4RVXtHl+NE0duKqduvOS5caUlvDvlOP497VTe\nnXIcQFx/ERBXjIoL4lfFS/R4U7EtktHuSXZo0JcgT6QoP4dVy5YyfWEX1ldUMWTSg3xT3Jvdu+pv\np6KikGiLEusvcbKNSSVCyVZK7045Lu75pKvmLK/n6M0RuPG0Q5PO4xRbwRjtnmQRk58d3qfRVqKu\n4hvWz5rKdRefzZeff4oC2/L2ruc/iWV9RVXCLUqsv8RJlm2qVP/mhLx9OZL0Oh1SCoyI7C0iD4rI\nS5HrASJykWsWGEaGcFreIFnE5I/jBu0pcaCq7Fj5Cusf/n9sXvcp3U6ZRIeuqfsHKeGVxVlDSxpt\nUWJXFLE+FWgcik7kwI0lWcg7HtMXrqE2WH+FVhvUrHYVeAR4GLgucv0v4EnCaf6G0SJxErVxwshp\nr7Orpo6QKpufv41dn7xNXp+BdD/1Kjp02cvxOOUVVTz9QXkjUWnIuNKSPc83pyHa5JP6O44sgftJ\nfg1xIjDdVXWOiEwFUNU6EWl67rJhZIC5y8rrhXeLAn5uOv3QhL6ISU8uZ+ozK8n3+6jYVcs+RQEK\n/Dnsqg3FHT/qExERcnseSG7Pg+j8o58gOU2PwER9IU4FLlZsnNLwrFEqYXLiG0oHJwKzU0S6EYmY\nicgIoNKV2Q3DIfE+zQEm/2NFvSV+RVVto7M1DamqDVEVEZTyiqqEPodQTRXb3niQwAHDKThwOF0O\nPyvtnyMb9ViaIkyTT+rf6Hfo92X3LNJVwPPAASLyLtADONuV2Q3DAfG2O795cjkBf04j/wFAbUgT\nRn/iEYwjRrvLP2Hz/Duo2/YNHTrvBQcOj/takaZVf8t2PRZHNLTfvaNIqZ28qvpP4FjgSOAy4FBV\nXemeCYaRnHjbHYWE2xr4PjGtqWiwjoq3HuebWb9Fg0H2/vmtdDnip4nv1/C2rMD//VupuMDPeSP6\npowatQSmL1zTaLVXG8qik1dEfkW4INTHketiEfmZqv6PKxYYRgqas60QwvkcTWXXZ0uofO9JCgce\nT9fRl5KTV5jyNRVVtQT8Pu4+Z0i9rcmwfbs22UmbbVqCk/cSVf1L9EJVt4nIJYAJjJEVmnOORoGd\nNc5iEaohard8TW73vhQcfCR7n/tnAr0PbdJOIZ4DtzlO2myTaSevk0S7HInpnyAiPiDXjclF5CER\n+VZEPnJjPKNtMvmk/hkrHFT33Wa+nXMj3zx2FXXbNyMiFPYZyF3nDOHLaadSFHCeMp/tgtpu4CQB\nMB2cCMxCYI6IHC8ixwGzgQWuzB7OsRnj0lhGG2VcaQnnjujrusjsXP02Gx66gt3lqygedSG+Tt0I\n+H3c8dPBe1YelU0oW9AiHbgpSHRg0q2Vl5Mt0u8IO3f/k/DW9mXg725MrqpvRVqWGEZS/jhuEMP2\n7crVc1akXXFNQ0G2zL+LnaveJLfXwXQfezX+riX4RBqdcHa6PXPbgducJLvmksmtnKiL5fGaZUBY\nYOap6sAEz8f2RRq6du3a7BlnZIXom6m8oiqckh95vLjAz42nHZq09GNz2frqfeQEOtPliJ9SXJjP\n7rpQvTEFOHdE37g1VxpSFPAjwp7EvXTFIN7PGPD7XF1ZpIuIfKCqw1Ldl3AFIyJzVPWnIvIhcSLj\nqpq8kYtLqOr9wP0Aw4YN81YNDddIJCqxf+Btu2qZ/I8Ve66jn+hFBX7yOuQ0qeqa1tWw7a3HKPzB\nseT1Ooji4y9FRAj4fYg0LqytwKzF4fap+f6cpAITK07NPZIQS7IT0S1FYJySbIsUbUY/NhuGGG2T\nRBm4sZ/QyT41aoPKdc9+WC8itG1XLX6fcHekKn5UqBJRs/ELNs+7ndrN6/AVdCGv10GIyJ7qcL95\ncnnc10VFJtWnmttikOnQcTZJKDCquiESMXpQVUdn0SajjZDowGFeh+QrgobECzfXBpWbX/iYZTec\nyLjSEvabMr9xQmooyPalz1Lx9kx8gU7sNf5mAvuHW4YI4QjK9IVrkgpIc5fM6YhBpkPH2SRpFElV\ng8AuEemSiclFZDbwHtBfRL62MhBti0RLfbcKSsfWXymKU4Ftx4evUbHoEQoOHE6vi/6yR1wAkHA/\noEzVqU1HDDIdOs4mTqJI1YSbo70C7Iw+qKpXpju5qv4s3TGMlks2l/TRWIWqEvxuCx06d6fjoOPx\nFXQhcOBwpEEFflWoTRHgiPUNNYV0xaCpJ6JbMk4EZn7kyzCaRKYr2Qf8OYyc9jrrI/Vng7sq2bJg\nBjXr19Drov/BF+hEwUGHN3lcIWz7qEN68PQH5Y2iOdW1wYTCU+KSGLSGLGAnpBQYVX1URHKBQwgL\n+hpVrcm4ZUarJ17xIzepC+keAdv12VK2vHQvod07KD7mF+Tkh88QlRQFeHfKcXF9NPGI3h8l3nmi\nsrVbmRmJMMVy3oi+/HHcIFd+traCk8OOpwD3AZ8TFvf9ROQyVX0p08YZrY+GUaOzhpY4isSkorjA\nX6+olEjYB6PBWra+ch87VizA36Mfe0/4I7k9+gH1eyw7WU3F29rEW0lEr2cv+YqghktD/OzwPiYu\ncUiZaCcinwBjVfWzyPUBwHxVPSQL9tVj2LBhWlZWlu1pDYfESxDLERq1Jm0qPhE+v/WUeo9FVySq\nyqZnb8HftYSio85DOnzv7C0u8LPshhMT2ub3CYW5HaiscidBrj2RdqJdDN9GxSXCF8C3zbbMaLPE\na7GarrhA495EtbW11JXNIXTgMfiLetJj3NS4JSwrYqJMbclx6jaZPJbgRGA+FpEXgTmEfTDjgfdF\n5EwAVX3GFUuMVs3cZeWuhZ8bUhIT8p3x9JtM/fVl7Cz/F0VBH10OPythfVwlXLQ7+oZpK45TN3Gr\nOHoinJymzgc2Eq5q92NgE9AVOA3L8m21OG3p4RS3KqDFY/JJ/QmFQlzyuz/w6wknUbX1G3qMu9ZR\njdzyiiomPbmc6+d+mDH7WjOpWtqmi5Mo0gWuzGS0GDLxqZXJnJdxpSXcfffd/P22GwjsP4xuJ/8a\nX8fiJo0xc/E6hu3b1VYwDcj0sQTr7NgOcetTK3YVlCOZKQkVqt4BwMUXX0z3U35Dj7NvbLK4RMnk\nKqu1Ei8DOtnjTcUEph3ixqdWdBUUbbIer0aLP0fw+5onPMHqHWx6YTobHr+GJ9/7jI4dO3Lw0WMb\nZeRCOHfi7kgFumS0xsOCmSZRENmtKi4mMO2QprYXjcfNLzSOGEE4pBytjDZ9/GC6NuOTsOrL5Wx4\n6Ap2rX6bwgHH8qeXPmXktNcT5rFEW7NCODSdiNZ4WDDTJHLMu+WwT1YP5qpkL1TVO12xwMg6TW0v\n2pC5y8oTNnoPqfLvyErihDvfZON3zpO+ta6WbYse4buy5+jQtTc9z7+dvF4HU1kTorIm+eojujq5\n8bRDufqpFY16HcUm3Rnfk6h/lM+lLW8yJ2+nyL/9gR8Rbr4G4ejRW67MbnhCujkhyXwZCvSbMp+8\nDjnsrkvctyguIuwuX0WnH46l6Me/JMef7/il0dVJ9GeI107WHLyNSVR+NN2ypFGS1YO5GUBEXgZ+\nqKrfRa5vAp5yZXbDM1LlhCQqFJWquFMUp+KioSDfffAChQOPwxfoTM+f/xnp0LSmFQ1XX5bv4pyS\nBEcoSlzaTjpJtOsLxK5za4B+rsxutEjihbEnJaj6lg61Fd+wZd6d7C5fBSJ0HvYTx+LiEyGkahm5\naZLudjkVTgTmcWCpiDxLeAV8BvCYG5OLyBjgHsAH/F1Vp7kxrpEe8cLYbqKq7Fj5CttefwAkh25j\nr6ZwwI8dv76lFcBuzWT6CIWTRLtbROQl4OjIQxeo6rJ0J46U4/wLcALwNeHjB8+r6qp0xzbSI9Ph\n3O2Ln6LircfI63sY3U+dFG4un4KSooCdIcoQmdxSOlnBABQA21X1YRHpISL7qeq/05x7OPCZqn4B\nICL/C/wEMIHxgFifS06CyEK6aF0t0sFPx0EnIP58Og0di0jqTIniAn+9Gi1G6yHlX1dEbiTcfG1q\n5CE/MNOFuUuAr2Kuv4481nD+S0WkTETKNm3a5MK0RkOcJM2lQ6imii0LZrBxzu/RUBBfx2I6Dzvd\nkbhAOPRstE6crGDOAEqBfwKo6noR6ZT8JY6IF2iP13/J+iJliNjeRJmi+uvVbJl/B3UVG+l8+FnN\nShG17VDrxYnA1KiqiogCiEihS3N/DfSJue4NrHdpbIPkdT6a2yHRnwO1DiLQGqyj4t3ZbF/8FL7O\nPdj757eS3ydu886kuBUuNbzByRp1jojcBxSJyCXAq7jTm/p94CAR2S9S83cC3yfzGWnScNtTXlHF\nb2LKFjQ3UuREXCDcSXHX6kUUDjyOfS6Y0Sxxsezb1o+TKNLtInICsJ1wVu8NqvpKuhOrap2IXAEs\nJBymfkhVP053XCNMPAGJdioctm/XjESKVEPs/PA1CgccS05eAT0n3o0vv2OzxhKB6eMH2/aoleOk\n6PefVfV3wCtxHksLVX0ReDHdcYz6zF1WnvJgYJeA39UKdHXbN7PlxbupXrscDQXpNGRMs8UlaqiJ\nS+vHyRbphDiPney2IYY7RLdGySivqGJ7tXvisnP1W2x46FfsXr+ariddQcfBJ6U9pp18bhskO039\nn8DlwAEisjLmqU7A/2XaMKN5OPWtuFGMG6Di3dlUvjOL3H36033s1fiL90l7zNbaJtVoTLIt0hPA\nS8CtwJSYx79T1a0ZtcpwRMMo0ahDemQ05ByLagiRHAr7H4VIDp1HnJ2w+HYqRh7QlVUbvttz+jmv\ng5Upais46Ys0Avg45jR1J2CAqi7Jgn31sL5I39PcMHO6hGp3U7HoUYK7Kulx+mRXxiwK+NldF2p0\n4C7eeaNMttgwnOO0L5KTj4q/AjtirndGHjM8JNMHEuOx+5vP+ObRSXz3wfP4CjqjIXfmr6iqdVQj\nOF7ofeozH6bdEcHIHE4S7URjljmqGhIRp2eYjAyRzfqyGgqyfcnTVLwzC19BF/b66X8R2O+HGZ+3\n4c+YrFi5rWJaJk6E4gsRuZLvVy2XE+7uaHhAdIuQzTMToartbF/6LAUHH0nXEy/HF3DjpEhqGkaS\nMt1iw3AfJwLzH8C9wPWE0yheAy7NpFFG4opyk59aQa1bIaAkqCpVny0hcOBwfIXF9LpgBr5O3eJW\n9U+G3yfUBptub7xIUqIG9hbSbrk4yeT9lnAav5ElEjVGEzShuAhxToo2k+DObWxZMIOqz5bS/bTJ\nFA44lg6duzdrrMLcDhTmddgjlLtq6uIWDC8u8FOQ2yGp8zbT1dcM90mWB/NbVb1NRGYQ/5TzlRm1\nrB2TyNeQDLfEZdenS9iy4F5Cu3dRfNzFFPzg6NQvSkJlVS3Lbzxxz3W86FfA7+PG01IX5bYG9q2P\nZCuY1ZF/LS6cZbzyKVS8PYvK/5uNf6/92XvCn8jtsW/aYzbcvqQrEg1fH400mci0TJJ1FXgh8u+j\n2TPHgMS+hkTkiDuZufn9BqOhWoqOOhfxudM6NN72JZ0SjZnoq21kjmRbpBdIsvJW1dMzYpHBqEN6\nMHPxOsf3h7R5PhgN1lLx7mwIBSn+8QXk9xnYrLIKiSgu8Lv+prdQdesi2Rbp9si/ZwI9+b5M5s+A\nLzNoU7vnjU+aXho0VlyKC/zU1IXYWZPYb1OzeR1b5t1BzcbP6XjYiahqkyNEyfD7JCOlLi1U3bpI\ntkVaBCAif1DVY2KeekFErLNjBkn3zVJdG0ISrGdUQ3z3wQtse/MRcnID9DjjOgoOPqJZ80R7ExUV\n+KmuDVIVqUZVXOB35LRtDhaqbl04yYPpISL7x1T/3w/okc6kIjIeuAn4ATBcVdu9I9nNqv7JIk51\nFRvZ9uYjBPoNodvJV+IrLG72PEFVvoz0oc4WFqpuXTgRmN8Ab4pINHu3H3BZmvN+RHjrdV+a47QJ\nGjouM9EypPrrVeT3HoC/uBe9Jt6Nv3vftLdEQtj2bPo+LFTdunCSaLdARA4CDok89Imq7k5nUlVd\nDbi652/NpDq4KMCRB3Tlyy1VTS7HEKzewdaX/4ddq99ir7NvInDAMFfCz/B9dbxsv7mt93TrwUlf\npAJgMnCFqq4A+orI2Ixb9v38bb4vUiqfiwJfbqni3SnHcfc5Q/DnOBPmqi+Xs+HBX7FrzbsUHX0+\n+fuVOrbJqfabc9VIhpMt0sPAB0DUE/g18BQwL9mLRORVwtGnhlynqs85NbA99EVykvcSfSNPX7jG\n0VmkirdnUvl//0uHrr3pedbvyet5YJNscrpLM+eqkQwnAnOAqp4jIj8DUNUqcbC3UdXRaVvXTojn\nuGxI9I3sdMXg774vnYaeRtGxE8nx57tiZ0PMuWqkwlHjNREJEEm1EJEDgLR8MEZ9Yh2X5RVVjZLm\nYt/IiVY70Zotkhug89DTKPzB0RSmeY4o4PfVEz2/TyjM7UBlVW1azlWrStd+cCIwNwILgD4iMgsY\nCfwynUlF5AxgBuFw93wRWa6q6Zeib8XEOi6TvQHjrXbqtm1g8/w72V2+msKBxzue0ydCbgfZk78S\nS1HAz02nH+q6EFiqf/siaU3eyFaoN7ALGEE4oLFYVTdnx7z6tKWavOl8iu/pKb1tFx0+fYN1L/2V\nOs2h24mXUzjgWEdjCHDXOUOAxjVm/DmSsaZnI6e9HncFVlIU4N0px7k+n5EZnNbkTbqCifSknquq\nQ4H5rlnXzkn1KZ5KfKKrnRUrVlBaehejRo1iTf/z6dDZWf6jAOeO6FtvzGTzubmlSeRDKq+oynpO\njZF5nGyRFovIj1T1/Yxb005IdmAPiCs+ZWu38sYnm1hfUUVRzbfceP6JjCsdzKJFixg5ciRH3/am\noxyZ4gI/quEWsm98smmPWCR6Y7u9pUkWMbOtUtvDSVeBUYRF5nMRWSkiHzZoxGY0kWSf4lfPWRFX\nfGYtXsdXG7ew+aV7WX7XRUy6dw5zl5Vz9NFHk5OTw+ST+hPwJ+9LVBTwU10boqKq1nFV/lRi2FSS\n2ZnOuEbLxMkKxtrEukyyT/FExwSqvl7Flnl3ULd9E52PGI92369eFq2TSJRI43NKqUoduH16OTrP\npCeXuzqu0TJJuIIRkXwRmUQ4i3cMUK6qa6NfWbOwDeJktRFLxTtPsPGJKSDC3j+fRvExv0B8/kZv\nxnGlJbw75Ti+nHYqd50zhJKiAELYgXrrmYOoiFMLF0i6tUqUSJdOgt240hJKMjCu0fJItkV6FBgG\nfEh4FXNHVixqg8xdVs7Iaa+z35T5jJz2OgC3njkIn8N8/Jy8AjoOGk2vX95Lfu8Bex5P9maMis2/\np53Ku1OOY1xpScL7o4cW4xFPDN1IsMvUuEbLItkWaYCqDgIQkQeBpdkxqW2RyEl665mDCCXYDqmG\n+K7sBXydu1PYfySdhv2k0cHQeG/GRK1Ooo8VFcQvg5ns0GKmTi/bqej2QcI8GBH5p6r+MNG1F7TG\nPJhkeR/QeHtSt30TW168i+q1KykceDzdT/1N3NfGCyU3TMDz5wgIjvoSCfDvLNd2MVovbuTBDBaR\n7dHxgEDkWginyHR2wc42TzIn6V3nDKmX5LZz1ZtsefmvEAoy/PwpbOw1st5rEjWEh/jRnqY0aDPf\nh5EJkpXMdO6FNBKSrMTjuNISbn7hY7btqqX664/Z/MLt5O1zCN3GXs23xb3q3S/AWUNLEibipRN9\nMd+HkSmc5MEYaZDKmbl54zcA5Pc+lB5nXMve5/4Zf3GvRhV1FZi/csOerVB5RVW9XJYuAedtRooL\n/I0iTOb7MDKBkzwYIw3GlZZQtnYrs5d8RVAVnwhnDS3hxP7FXHnllax/4H56Trwbf7c+FBx8ZNKx\ntu2q5bpnG5d1qKoNku/PaXz6OY4PxmkXRcNwA1vBZJi5y8p5+oPyPQl0QVVmzXuTgw8dzIwZMzjl\n7PPo2K3xdigRiVqRVOyq5dYzB9VbmUwfP5jpZw+21YrhGbaCyTANna+V782h4p1Z5HYs5pVXXmH0\n6NHMXVa+xxcDkO/PiVtCIRlRn06yULNhZBsTmAzT0PkarNpOQf+j6HbifzJ69PdF/6pjBKWp4gLx\nW7Qahtd4IjAiMh04DagBPgcuUNUKL2zJNL265LNm0Vz83fuS33sAxT++AMnx1UuVT9RVoOF5okTt\nYYsC7rdoNQw38MoH8wowUFUPA/4FTPXIjoyyceNGal+axtaF/82Ola8AIDm+RmHhRCFmJSweUf/J\nuSP6xo1I3XS6+y1aDcMNPFlEZx0EAAAJU0lEQVTBqOrLMZeLgbO9sCOTzJ07l0suuYQdO3Zw0eSb\nWVV0JBu2746bEp/sdPXuuhB3nTNkz/3D9u2aVnq91cM1sknSkplZMUDkBeBJVZ2Z4PlLgUsB+vbt\nO3Tt2pZ/kHvBggWcfPLJlJaWMnPmTAYMGJD0/nhp/rG4VU4y3jzJsoMNIxFOjwpkbIskIq+KyEdx\nvn4Sc891QB0wK9E4qnq/qg5T1WE9eqTVEjvjVFZWAnDCCSdw3333sXjx4pTiAuEoz61nDkr4vFs1\nUtwuHmUYqciYwKjqaFUdGOfrOQARmQiMBc5Vr5dRaVJTU8PUqVM5+OCD2bBhAz6fj0svvZTc3FzH\nY2SjRorbxaMMIxWeOHlFZAzwO+B0Vd3lhQ1u8dFHH3H44Yczbdo0Tj/9dDp27NjssTJdIyUTxaMM\nIxleRZH+G+gEvCIiy0Xkbx7Z0WxUlbvuuothw4axfv16nnvuOR544AE6derU7DGjW6VMZd5akScj\n23ju5G0KLakejKoyYcIEqqureeCBB9hrr728NskRFkUy3MCpk9cEpgmoKk888QRDhw7lkEMOobq6\nmry8vEbV5gyjreN5FKmtsXXrViZMmMB5553HPffcA0B+fn67EJeGNYWTtTkxjFjsLJIDFi5cyIUX\nXsi3337Ln/70J3772996bVLWsF7SRjrYCiYFTz/9NGPGjKGoqIilS5cydepUfL72U+zPcmeMdDCB\nScDu3bsBOOWUU7jlllsoKyujtLTUY6uyj+XOGOlgAtOAuro6/vCHPzB48GB27NhBIBDg2muvJRBo\nn7kiljtjpIMJTAyffvopRx11FDfccANDhw4lGIx/Nqg9YbkzRjqYk5dw+Pn+++/nqquuIjc3l9mz\nZzNhwgSvzWoRWIM0Ix0sDwYIBoOMGjWKvLw8Hn74YXr37u36HIbRlnCj8Vqb57nnnmPEiBHsvffe\nPP/883Tu3JmcHNs1GoZbtMt30/bt27nwwgsZN24ct912GwBFRUUmLobhMu1uBfP222/zi1/8gnXr\n1nH99dfz+9//3muTDKPN0q4EZvbs2Zx77rnsv//+vPPOOxxxxBFem2QYbZp2sScIhcJtQE488USu\nuuoqli9fbuJiGFmgTQtMKBTizjvvZNSoUdTV1dGtWzduv/32tIpCGYbhHK8q2v1BRFZGik29LCL7\nuD3HunXrGD16NFdffTVdu3Zl586dbk9hGEYKvFrBTFfVw1R1CDAPuMGtgVWVmTNnMmjQIN5//30e\neughnnnmGbp06eLWFK5ipRCMtoxXfZG2x1wWEr9hYbOoqanhlltu4bDDDuOxxx5jv/32c2to17FS\nCEZbx7MokojcAvwCqARGJbkvti9SynHz8vJ49dVX6dmzZ4svq5CsFIIJjNEW8Kwvkqpep6p9CPdE\nuiLROM3pi1RSUtLixQWsFILR9snYCkZVRzu89QlgPnBjpmxpqSRqGWulEIy2gldRpINiLk8HPvHC\nDq+xUghGW8crH8w0EekPhIC1wH94ZIenWCkEo63jVRTpLC/mbYmMKy0xQTHaLG06k9cwDG8xgTEM\nI2OYwBiGkTFaVclMEdlE2Cmciu7A5gyb45SWYovZ0ZiWYktLsQOc27KvqqZMTGtVAuMUESlzUi80\nG7QUW8yOxrQUW1qKHeC+LbZFMgwjY5jAGIaRMdqqwNzvtQExtBRbzI7GtBRbWood4LItbdIHYxhG\ny6CtrmAMw2gBmMAYhpEx2qzAZKPur0M7povIJxFbnhWRIi/siNgyXkQ+FpGQiGQ9LCoiY0RkjYh8\nJiJTsj1/jB0Pici3IvKRVzZE7OgjIm+IyOrI3+XXHtmRLyJLRWRFxI6bXRtcVdvkF9A55vsrgb95\nZMeJQIfI938G/uzh7+QHQH/gTWBYluf2AZ8D+wO5wApggEe/h2OAHwIfefW3iNjRC/hh5PtOwL+8\n+J0AAnSMfO8HlgAj3Bi7za5gNIN1f5tox8uqWhe5XAz09sKOiC2rVXWNR9MPBz5T1S9UtQb4X+An\nXhiiqm8BW72Yu4EdG1T1n5HvvwNWA1k/Wq9hdkQu/ZEvV94vbVZgIFz3V0S+As7Fxc4FaXAh8JLX\nRnhECfBVzPXXePBmaqmISD+glPDqwYv5fSKyHPgWeEVVXbGjVQuMW3V/M21H5J7rgLqILRnDiS0e\nIXEesxwJQEQ6Ak8DkxqsvLOGqgY13EaoNzBcRAa6MW6r7k2tLaTubyo7RGQiMBY4XiMb3UzRhN9J\ntvka6BNz3RtY75EtLQYR8RMWl1mq+ozX9qhqhYi8CYwB0naCt+oVTDJaSt1fERkD/A44XVV3eWFD\nC+F94CAR2U9EcoEJwPMe2+QpIiLAg8BqVb3TQzt6RKObIhIARuPS+6XNZvKKyNOEIyZ76v6qatbb\nJorIZ0AesCXy0GJV9aQGsYicAcwAegAVwHJVPSmL858C3E04ovSQqt6Srbkb2DEb+DHh0gQbgRtV\n9UEP7DgKeBv4kPD/U4BrVfXFLNtxGPAo4b9LDjBHVf/LlbHbqsAYhuE9bXaLZBiG95jAGIaRMUxg\nDMPIGCYwhmFkDBMYwzAyRqtOtDPcRUS6Aa9FLnsCQWBT5Hp45AxRtm1aCJwdOatjtDIsTG3ERURu\nAnao6u0NHhfC/29CcV/o3vxZmcfILLZFMlIiIgdGzjP9Dfgn0EdEKmKenyAif498v7eIPCMiZZEa\nIyPijHdxpDbOwkh9mOsTzNNLRL6OyTK9IFJXZ4WIPOx0PsM7bItkOGUAcIGq/oeIJPt/cy9wm6ou\njpwQngfEOzg3PPJ4DfC+iMwDdsTOAxBeyICIDCZ85OJIVd0qIl2bOJ/hASYwhlM+V9X3Hdw3Gugf\nFQagWEQCqlrV4L6FqroNQETmAkcBC5LMcxzwpKpuBYj+24T5DA8wgTGcsjPm+xD1yy/kx3wvOHMI\nN3T+Ra93NrwxZtx4DkOn8xkeYD4Yo8lEHK/bROQgEckBzoh5+lXgV9ELERmSYJgTRaRIRAoIV7Z7\nN8W0rwITolujmC2S0/kMDzCBMZrL7whvaV4jXOslyq+AkRFn7CrgkgSvf4dwnZ5lwGxVXZ5sMlVd\nCdwGvBWpvDa9ifMZHmBhaiPriMjFwEBVneS1LUZmsRWMYRgZw1YwhmFkDFvBGIaRMUxgDMPIGCYw\nhmFkDBMYwzAyhgmMYRgZ4/8DMj8ujD+T9cwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a1c1f1650>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#还可以观察预测值与真值的散点图\n",
    "plt.figure(figsize=(4, 3))\n",
    "plt.scatter(y_train, y_train_pred_lr)\n",
    "plt.plot([-3, 3], [-3, 3], '--k')   #数据已经标准化，3倍标准差即可\n",
    "plt.axis('tight')\n",
    "plt.xlabel('True price')\n",
    "plt.ylabel('Predicted price')\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在y的真值大的部分预测效果不好"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 正则化的线性回归（L2正则 --> 岭回归）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The r2 score of RidgeCV on test is 0.695268272444\n",
      "The r2 score of RidgeCV on train is 0.754787797155\n"
     ]
    }
   ],
   "source": [
    "#岭回归／L2正则\n",
    "#class sklearn.linear_model.RidgeCV(alphas=(0.1, 1.0, 10.0), fit_intercept=True, \n",
    "#                                  normalize=False, scoring=None, cv=None, gcv_mode=None, \n",
    "#                                  store_cv_values=False)\n",
    "from sklearn.linear_model import  RidgeCV\n",
    "\n",
    "#1. 设置超参数（正则参数）范围\n",
    "alphas = [ 0.01, 0.1, 1, 10,100]\n",
    "#n_alphas = 20\n",
    "#alphas = np.logspace(-5,2,n_alphas)\n",
    "\n",
    "#2. 生成一个RidgeCV实例\n",
    "ridge = RidgeCV(alphas=alphas, store_cv_values=True)  \n",
    "\n",
    "#3. 模型训练\n",
    "ridge.fit(X_train, y_train)    \n",
    "\n",
    "#4. 预测\n",
    "y_test_pred_ridge = ridge.predict(X_test)\n",
    "y_train_pred_ridge = ridge.predict(X_train)\n",
    "\n",
    "\n",
    "# 评估，使用r2_score评价模型在测试集和训练集上的性能\n",
    "print 'The r2 score of RidgeCV on test is', r2_score(y_test, y_test_pred_ridge)\n",
    "print 'The r2 score of RidgeCV on train is', r2_score(y_train, y_train_pred_ridge)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 可视化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XuUlPWd5/H3t2/c7zTIHVTQoCJi\nCxiTGI0mEBNREyMqjpN11nVXz2Y2J1nNyaw5m13PbJLdzDnZmJlkZrLJ2CjxmmCC8Z6oiQU0Cgje\naNHqboHuBrlD37/7Rz1tHpru6mqrn7p+Xuf06aee+j1PfeuB7k8/l3q+5u6IiIh8VCXZLkBERPKb\ngkRERNKiIBERkbQoSEREJC0KEhERSYuCRERE0qIgERGRtChIREQkLQoSERFJS1m2C8iEiRMn+uzZ\ns7NdhohIXtm0adNed6/sb1xRBMns2bOpqanJdhkiInnFzOKpjNOhLRERSYuCRERE0qIgERGRtChI\nREQkLQoSERFJi4JERETSoiAREZG0KEhERArQrgPH+V9PvEnz4dbIX0tBIiJSgB7YUMdPX3iHlvbO\nyF9LQSIiUmDaO7tYs7GeS86YxIzxwyN/PQWJiEiBeWp7I82HW7lp6ayMvJ6CRESkwNwXe4/p44bx\nqXn93m9xUChIREQKSG3TYWI7P+CGJTMpLbGMvKaCRESkgFTH6qgoLeErVTMy9poKEhGRAnGsrYNH\nNjWw/JxTmDhySMZeV0EiIlIg1m7exeHWDlZl6CR7NwWJiEgBcHeq18c5Y/IoqmaNy+hrK0hERArA\nloaDbHv/EKsunIVZZk6yd1OQiIgUgPtejjOiopSrz5uW8ddWkIiI5LkDx9r47dZdXHXeNEYOKcv4\n6ytIRETy3MObGmjt6Mr4SfZuChIRkTzW1eVUx+JUzRrHx6aMzkoNChIRkTz2p3f28t6+Y1nbGwEF\niYhIXquOxRk/ooLl55yStRoiDRIzW2Zmb5lZrZnd1cvzt5nZa2a22cxeMrP5wfwJZva8mR0xsx/3\nWOb8YJlaM/uRZfo6NxGRHLH74HGefr2Rr1TNYEhZadbqiCxIzKwUuBdYDswHru8OipD73f0cd18I\nfB/4YTC/BfhvwDd6WfU/ArcCc4OvZRGULyKS8x7YUI8DNy6ZmdU6otwjWQzUuvtOd28D1gArwgPc\n/VDo4QjAg/lH3f0lEoHyITObAox295fd3YF/A66K8D2IiOSk9s4u1myo4+J5lRlpXpVMlBccTwPq\nQ48bgCU9B5nZ7cDXgQrg0hTW2dBjnZn/9I2ISJY983ojTYdb+fssnmTvFuUeSW/nLvykGe73uvtp\nwJ3A3w3GOgHM7FYzqzGzmubm5n6LFRHJJ/fF4kwbO4xPnzEp26VEGiQNQPiG+NOBXUnGr6H/w1QN\nwXr6Xae7/8zdq9y9qrIyM13CREQy4Z3mI/z5nX0ZbV6VTJRBshGYa2ZzzKwCWAmsDQ8ws7mhh1cA\nO5Kt0N13A4fNbGlwtdZfAb8Z3LJFRHLb6lgd5aWW0eZVyUR2jsTdO8zsDuBJoBT4ubtvN7PvAjXu\nvha4w8wuA9qB/cDN3cub2XvAaKDCzK4CPuvurwP/EfgFMAx4IvgSESkKx9s6eXhTPcvOnkLlqMw1\nr0om0rt7ufs6YF2PeXeHpr+WZNnZfcyvAc4epBJFRPLK41t2cailg1VZvuQ3TJ9sFxHJI9Xr48yb\nPJLFc8Znu5QPKUhERPLElvoDbG04yKqlmW9elYyCREQkT1TH4gzPUvOqZBQkIiJ54OCxdh7fuosV\nC6cxamh5tss5gYJERCQPPPxKAy3tXaxamjsn2bspSEREcpy7szoWZ9HMsZw1dUy2yzmJgkREJMf9\n+Z197Nx7NKvNq5JRkIiI5LjqWJxxw8v5/DlTsl1KrxQkIiI5bM/BFp4KmlcNLc9e86pkFCQiIjls\nzcY6OrucG3Lok+w9KUhERHJUe2cXDwTNq2ZNGJHtcvqkIBERyVHPvtFI46HWnD3J3k1BIiKSo6pj\ndUwdM5RLz8x+86pkFCQiIjno3b1Heal2L9cvzo3mVckoSEREctDqWJyyEuO6xbnRvCoZBYmISI5p\nae/koU0NfO7sU5g0ami2y+mXgkREJMc8vmUXB4+3s2pJbp9k76YgERHJMdXr6zh90kiWnpo7zauS\nUZCIiOSQ1xoOsqX+AKuWzMyp5lXJKEhERHJIdSzOsPJSrjl/erZLSZmCREQkRxw83s5vtrzPioVT\nGZ1jzauSUZCIiOSIRz9sXpUfJ9m7KUhERHKAu1Mdi7NwxljOnpZ7zauSUZCIiOSAl3fu453m3G1e\nlYyCREQkB6yO1TF2eDlfWJCbzauSUZCIiGRZ06EWnty+h2vPn56zzauSUZCIiGTZmo31dHQ5N+TJ\nJ9l7UpCIiGRRR9C86pNzJzJnYu42r0pGQSIikkXPvtnE7oMteXmSvZuCREQki6pjcaaMGcpncrx5\nVTKRBomZLTOzt8ys1szu6uX528zsNTPbbGYvmdn80HPfCpZ7y8w+F5r/XmiZmijrFxGJ0nt7j/Li\njr2svGAmZaX5+3d9WVQrNrNS4F7gcqAB2Ghma9399dCw+939n4LxVwI/BJYFgbISOAuYCjxjZvPc\nvTNY7hJ33xtV7SIimXD/hjrKSoyVedC8KpkoI3AxUOvuO929DVgDrAgPcPdDoYcjAA+mVwBr3L3V\n3d8FaoP1iYgUhJb2Th6sqeezZ01m8ujcb16VTJRBMg2oDz1uCOadwMxuN7N3gO8D/zmFZR14ysw2\nmdmtfb24md1qZjVmVtPc3JzG2xARGXy/27qbA8fyp3lVMlEGSW830veTZrjf6+6nAXcCf5fCshe5\n+yJgOXC7mX2qtxd395+5e5W7V1VWVg68ehGRCFWvj3Nq5QguPG1CtktJW5RB0gCED/xNB3YlGb8G\nuKq/Zd29+3sT8Bg65CUieWbb+wd5te4Aq5bMypvmVclEGSQbgblmNsfMKkicPF8bHmBmc0MPrwB2\nBNNrgZVmNsTM5gBzgQ1mNsLMRgXLjgA+C2yL8D2IiAy61evjDC0v4Ut51Lwqmciu2nL3DjO7A3gS\nKAV+7u7bzey7QI27rwXuMLPLgHZgP3BzsOx2M3sQeB3oAG53904zmww8FiR4GYmrvn4f1XsQERls\nh1ra+fWru1hx7jTGDMuf5lXJRBYkAO6+DljXY97doemvJVn2HuCeHvN2AucOcpkiIhnz6KYGjrd3\n5vUn2XvK30/AiIjkGXenen0d504fwznT86t5VTIKEhGRDFn/7gfUNh3hxgLaGwEFiYhIxlTH4owZ\nVs4XF0zNdimDSkEiIpIBTYdb+P22PXz5/OkMq8i/5lXJKEhERDLgwaB51Y1LZma7lEGnIBERiVhn\nl3P/+jo+cfpETq0cme1yBp2CREQkYs+92cSugy2sWlp4eyOgIBERiVx1LM7k0UO47GOTs11KJBQk\nIiIRqtt3jBd2NHP94vxuXpVMYb4rEZEcsXpDnBIzVl5QmIe1QEEiIhKZlvZOHtxYz+Ufm8wpY/K7\neVUyChIRkYg8sW03+4+1F9R9tXqjIBERiUh1rI5TJ47g4wXQvCoZBYmISARe33WITfH93LBkJiUl\n+d+8KhkFiYhIBKrXxxlSVsKXC6R5VTIKEhGRQXa4pZ1fv/o+V547lbHDK7JdTuQUJCIig+yxV9/n\nWFthNa9KRkEiIjKI3J3qWJxzpo3h3Bljs11ORihIREQG0cb39vN24xFuKpK9EVCQiIgMqupYnNFD\ny/jiuYXVvCoZBYmIyCBpPtzKE9t286UCbF6VjIJERGSQPFhTT3unc+OS4jmsBQoSEZFB0d286uOn\nTeD0SYXXvCoZBYmIyCD4w1tNvH/geNFc8humIBERGQTVsTiTRg3h8vmF2bwqGQWJiEia6j84xh/e\nbmbl4pmUF2jzqmSK7x2LiAyy1evrKDHj+sUzsl1KVqQcJGb2CTP7ajBdaWZzoitLRCQ/tHZ08mBN\nPZ85cxJTxgzLdjlZkVKQmNl3gDuBbwWzyoHqqIoSEckXv9+2hw+OtnHThcV3kr1bqnskVwNXAkcB\n3H0XMKq/hcxsmZm9ZWa1ZnZXL8/fZmavmdlmM3vJzOaHnvtWsNxbZva5VNcpIpJJ1bE4sycM56LT\nJma7lKxJNUja3N0BBzCzEf0tYGalwL3AcmA+cH04KAL3u/s57r4Q+D7ww2DZ+cBK4CxgGfATMytN\ncZ0iIhnx5p5DbHxvPzcumVXwzauSSTVIHjSznwJjzezfA88A/9zPMouBWnff6e5twBpgRXiAux8K\nPRxBEFTBuDXu3uru7wK1wfr6XaeISKZUx+JUFEnzqmTKUhnk7v/bzC4HDgFnAHe7+9P9LDYNqA89\nbgCW9BxkZrcDXwcqgEtDy8Z6LDstmO53nSIiUTvS2sFjr7zPFxdMZdyIwm9elUyqJ9tHAM+5+zdJ\n7IkMM7Py/hbrZZ6fNMP9Xnc/jcTJ/L/rZ9mU1hnUfKuZ1ZhZTXNzcz+liogMzGOvvs/Rtk5WLZ2Z\n7VKyLtVDWy8AQ8xsGonDWl8FftHPMg1A+KLq6cCuJOPXAFf1s2zK63T3n7l7lbtXVVZW9lOqiEjq\n3J3VsThnTR3NwiJpXpVMqkFi7n4MuAb4v+5+NYmT3clsBOaa2RwzqyBx8nztCSs1mxt6eAWwI5he\nC6w0syHB51XmAhtSWaeISNQ2xffz5p7D3LR0FmbFe5K9W0rnSAAzswuBG4FbUlnW3TvM7A7gSaAU\n+Lm7bzez7wI17r4WuMPMLgPagf3AzcGy283sQeB1oAO43d07g0JOWmfqb1dEJH33xeKMGlrGlQuL\np3lVMqkGydeAu4BHg1/yc4Dn+lvI3dcB63rMuzs0/bUky94D3JPKOkVEMmXfkVaeeG0PNyyZyfCK\nVH+FFrZUt8IxoIvE5zZWkTjp3etJbhGRQvZgTQNtnV06yR6SapCsBr4BbCMRKCIiRaezy7l/Q5yl\np47n9En93tyjaKQaJM3u/niklYiI5LgX3m6m/oPj3LnszGyXklNSDZLvmNm/AM8Crd0z3f3RSKoS\nEclB1bE4E0cO4bPzT8l2KTkl1SD5KnAmibv+dh/ackBBIiJFof6DYzz3VhN3XHI6FWVq5RSWapCc\n6+7nRFqJiEgOe2BDHQZcv1gn2XtKNVZjusuuiBSrto4uHqyp59IzJzN1bHE2r0om1T2STwA3m9m7\nJM6RGODuviCyykREcsTvt+9h75Hibl6VTKpBsizSKkREclh1LM6sCcP55OnF27wqmVRvIx+PuhAR\nkVz01p7DbHj3A761/Myibl6VjC49EBFJYvX6RPOqa6tm9D+4SClIRET6cLS1g0dfeZ8vnDOF8UXe\nvCoZBYmISB9+vfl9jrR2cONSnWRPRkEiItILd6c6VsfHpoxm0Uw1r0pGQSIi0otX6g7wxu5Dal6V\nAgWJiEgvqmNxRg4pY4WaV/VLQSIi0sMHR9v43dbdXLNoGiOGqHlVfxQkIiI9PFRTHzSv0kn2VChI\nRERCurqc1evrWDxnPPMmq3lVKhQkIiIhL+xopu6DY9obGQAFiYhISHWsjokjK1h2lppXpUpBIiIS\neP/AcZ57s5HrLpih5lUDoC0lIhJ4YH0djppXDZSCRESERPOqNRvrufSMSUwfNzzb5eQVBYmICPDU\n63vYe6SVVWpeNWAKEhER4L6X48wYP4yL51Zmu5S8oyARkaK3o/Ew69/9gBsWz1Lzqo9AQSIiRW/1\n+joqSkv4StX0bJeSlxQkIlLUjrV18MimBj5/zilMGDkk2+XkJQWJiBS132zexeHWDn2SPQ2RBomZ\nLTOzt8ys1szu6uX5r5vZ62a21cyeNbNZoee+Z2bbgq/rQvN/YWbvmtnm4GthlO9BRApXonlVnDNP\nGcX5s8Zlu5y8FVmQmFkpcC+wHJgPXG9m83sMexWocvcFwMPA94NlrwAWAQuBJcA3zWx0aLlvuvvC\n4GtzVO9BRArb5voDbN91iFVqXpWWKPdIFgO17r7T3duANcCK8AB3f97djwUPY0D3ma75wB/dvcPd\njwJbgGUR1ioiRei+WJwRFaVcdd60bJeS16IMkmlAfehxQzCvL7cATwTTW4DlZjbczCYClwAzQmPv\nCQ6H/YOZ6eyYiAzY/qNt/Hbrbq5eNI2Ral6VliiDpLf9RO91oNkqoAr4AYC7PwWsA/4MPAC8DHQE\nw78FnAlcAIwH7uxjnbeaWY2Z1TQ3N6fxNkSkED28qYG2DjWvGgxRBkkDJ+5FTAd29RxkZpcB3wau\ndPfW7vnufk9wDuRyEqG0I5i/2xNagf9H4hDaSdz9Z+5e5e5VlZX6pKqI/EWieVWcC2aP48xTRve/\ngCQVZZBsBOaa2RwzqwBWAmvDA8zsPOCnJEKkKTS/1MwmBNMLgAXAU8HjKcF3A64CtkX4HkSkAL1U\nu5f39ql51WCJ7MCgu3eY2R3Ak0Ap8HN3325m3wVq3H0tiUNZI4GHgism6tz9SqAceDGYdwhY5e7d\nh7ZWm1klib2UzcBtUb0HESlM1bE4E0ZUsOxsNa8aDJGeYXL3dSTOdYTn3R2avqyP5VpIXLnV23OX\nDmaNIlJcdh88zjNvNPIfLj6NIWWl2S6nIOiT7SJSVLqbV92g5lWDRkEiIkWjvTPRvOrT8yqZMV7N\nqwaLgkREisbTrzfSdLiVm9S8alApSESkaNz3cpxpY4dx8bxJ2S6loChIRKQo1DYd4eWd+7hhyUxK\n1bxqUClIRKQorF4fp7zUuO6CGf0PlgFRkIhIwTvW1sHDmxpYfvYUJqp51aBTkIhIwXt8yy4Ot6h5\nVVQUJCJS8KpjdZwxeRQXzFbzqigoSESkoG2pP8Br7x9k1dKZal4VEQWJiBS0+2Jxhqt5VaQUJCJS\nsA4ca+PxLbu46rxpjBpanu1yCpaCREQK1sObGmjt6GLVEp1kj5KCREQKUqJ5VR3nzxrH/KlqXhUl\nBYmIFKQ/v7OPd/ceZdVS3eU3agoSESlI1bE440dUsPzsKdkupeApSESk4Ow52MLTbzRybdV0hpar\neVXUFCQiUnAe2FBHlzs3LtZJ9kxQkIhIQUk0r6rj4nmVzJyg5lWZoCARkYLy7BuNNB5q1SW/GaQg\nEZGCcl8s0bzqkjPVvCpTFCQiUjB2Nh/hT7X7uH7xDDWvyiAFiYgUjNXr6ygvNb6i5lUZpSARkYJw\nvK2Th2rq+dxZpzBp1NBsl1NUFCQiUhAe37qLQ2pelRUKEhEpCKtjceZOGsmSOeOzXUrRUZCISN7b\n2nCALQ0HWbV0lppXZYGCRETyXnUszrDyUq5epOZV2aAgEZG8dvBYO2u37OKq86YyWs2rskJBIiJ5\n7ZFXGmhp79JJ9iyKNEjMbJmZvWVmtWZ2Vy/Pf93MXjezrWb2rJnNCj33PTPbFnxdF5o/x8zWm9kO\nM/uVmVVE+R5EJHe5O9Xr45w3cyxnTR2T7XKKVmRBYmalwL3AcmA+cL2Zze8x7FWgyt0XAA8D3w+W\nvQJYBCwElgDfNLPuFmffA/7B3ecC+4FbonoPIpLbXn5nHzubj+q+WlkW5R7JYqDW3Xe6exuwBlgR\nHuDuz7v7seBhDJgeTM8H/ujuHe5+FNgCLLPE5RiXkggdgF8CV0X4HkQkh1WvjzN2eDlXLFDzqmyK\nMkimAfWhxw3BvL7cAjwRTG8BlpvZcDObCFwCzAAmAAfcvSPFdYpIgWo81MJT2xv5StUMNa/KsrII\n193bxdze60CzVUAVcDGAuz9lZhcAfwaagZeBjgGu81bgVoCZM9WzWaTQrNlQT0eXc8Ni/XxnW5R7\nJA0k9iK6TQd29RxkZpcB3waudPfW7vnufo+7L3T3y0kEyA5gLzDWzMqSrTNY/mfuXuXuVZWVlYPy\nhkQkN3R0dvHAhjo+Na+S2RNHZLucohdlkGwE5gZXWVUAK4G14QFmdh7wUxIh0hSaX2pmE4LpBcAC\n4Cl3d+B54MvB0JuB30T4HkQkBz37ZhN7DrWwaon2RnJBZIe23L3DzO4AngRKgZ+7+3Yz+y5Q4+5r\ngR8AI4GHgtsa1Ln7lUA58GIw7xCwKnRe5E5gjZn9TxJXff1rVO9BRHJTdSzOlDFDuVTNq3JClOdI\ncPd1wLoe8+4OTV/Wx3ItJK7c6u25nSSuCBORIvTu3qO8uGMvX798HmWl+kx1LtC/gojklfvXxykr\nMVaqeVXOUJCISN5oae/kwZqGRPOq0WpelSsUJCKSN367dTcHj7dz41KdZM8lChIRyRvVsTinVY7g\nwlMnZLsUCYn0ZLuISLoOtbRT23SEV+L72Vx/gO98cb6aV+UYBYmI5ITuwNjReJi3G4+wI5jefbDl\nwzEzxw/nmkXTk6xFskFBIiIZdailnR2NR6htSgTG242HqW06ckJgDC0v4fRJI1l66gTmTh7JvEmj\nmDt5JNPHDae0RHsjuUZBIiKRUGAUDwWJiKRlIIFx4akTOD0IjHmTRzFt3DAFRgFQkIhISroDY0fj\nYXY0JQJjR+MR9hxSYBQ7BYmInGAggfHx0xQYoiARKVrhwEhcJZU8MOZOHsXcSSOZN3kU08cNo0SB\nIQEFiUiBU2BI1BQkIgUiERiJkFBgSCYpSJJ4YEMdtU1HMMAMzCyYtsTj7vkYJcGD7nkloenuT+GW\n9LLcietNPFdSknjc5/p6Wy4YAyfODy9HsFxJj+VOeF89liP0ej3rJ3jf4eVKS4whZaUMKSthSHnJ\nX6bLSvRp5EFy8Hg7tU39B8bcSaM+DIx5k0cyd5ICQ6KhIEnixR3NvPD2XrrccQen+zu495gGvNfu\n8dKtIgiU3oLmhOfK+x6XeBya7m18H/Pz7RdoKoExrLz0hD0MBYZkg4IkiZ/ceP6AlwkHTM8A4oR5\noQDqOjmkuoLlOGldodfoJdy6HDhh3sl1fJTlktYRWm9Hl9PW2UVreyetHV3BVyet7aHpjq7gcWhM\neyeHWzrY29FGa0cnbaH53WPSVV5qAwqkilSD64Sg63s9fTVh6g6MtxuPJM5lNB3m7cbDNB5q/XCM\nAkNymYJkkHUf9gIoRT/gg8U9CKieIdRLILV2dP0liHod13uwHW3t4IOjXaHl//JcS0dn2nucicN+\nJwbS8bZOmg6fHBgXnT6RuZMSgTFv8iimjVVgSO5SkEheMOvemyiFLPQzcnc6uvykvaSTQik03ZbC\nHllpSeLktwJD8pmCRCQFZkZ5qVFeWsLIIfqxEQlTYysREUmLgkRERNKiIBERkbQoSEREJC0KEhER\nSYuCRERE0qIgERGRtChIREQkLeZFcKdBM2sG4h9x8YnA3kEsZ7CoroFRXQOjugamUOua5e6V/Q0q\niiBJh5nVuHtVtuvoSXUNjOoaGNU1MMVelw5tiYhIWhQkIiKSFgVJ/36W7QL6oLoGRnUNjOoamKKu\nS+dIREQkLdojERGRtChIejCzH5jZm2a21cweM7OxfYxbZmZvmVmtmd2VgbquNbPtZtZlZn1ehWFm\n75nZa2a22cxqcqiuTG+v8Wb2tJntCL6P62NcZ7CtNpvZ2gjrSfr+zWyImf0qeH69mc2OqpYB1vXX\nZtYc2kZ/k6G6fm5mTWa2rY/nzcx+FNS91cwW5UBNnzazg6FtdXfUNQWvO8PMnjezN4Kfxa/1Miba\n7ZXou62v7i/gs0BZMP094Hu9jCkF3gFOBSqALcD8iOv6GHAG8AegKsm494CJGdxe/daVpe31feCu\nYPqu3v4dg+eOZGAb9fv+gf8E/FMwvRL4VY7U9dfAjzP1/yn0up8CFgHb+nj+88ATgAFLgfU5UNOn\ngd9mYVtNARYF06OAt3v5d4x0e2mPpAd3f8rdO4KHMWB6L8MWA7XuvtPd24A1wIqI63rD3d+K8jU+\nihTryvj2Ctb/y2D6l8BVEb9eMqm8/3C9DwOfMbOoe+5m498lJe7+AvBBkiErgH/zhBgw1symZLmm\nrHD33e7+SjB9GHgDmNZjWKTbS0GS3L8jkeI9TQPqQ48bOPkfLlsceMrMNpnZrdkuJpCN7TXZ3XdD\n4gcNmNTHuKFmVmNmMTOLKmxSef8fjgn+kDkITIionoHUBfCl4HDIw2Y2I+KaUpWrP4MXmtkWM3vC\nzM7K9IsHh0TPA9b3eCrS7VWUzafN7BnglF6e+ra7/yYY822gA1jd2yp6mZf25W+p1JWCi9x9l5lN\nAp42szeDv6SyWVfGt9cAVjMz2F6nAs+Z2Wvu/k66tfWQyvuPZBv1I5XXfBx4wN1bzew2EntNl0Zc\nVyqysb368wqJW4ocMbPPA78G5mbqxc1sJPAI8Lfufqjn070sMmjbqyiDxN0vS/a8md0MfAH4jAcH\nGHtoAMJ/mU0HdkVdV4rr2BV8bzKzx0gcvkgrSAahroxvLzNrNLMp7r472IVv6mMd3dtrp5n9gcRf\nc4MdJKm8/+4xDWZWBowh+sMo/dbl7vtCD/+ZxHnDXBDJ/6l0hH95u/s6M/uJmU1098jvwWVm5SRC\nZLW7P9rLkEi3lw5t9WBmy4A7gSvd/VgfwzYCc81sjplVkDg5GtkVP6kysxFmNqp7msSFA71eYZJh\n2dhea4Gbg+mbgZP2nMxsnJkNCaYnAhcBr0dQSyrvP1zvl4Hn+vgjJqN19TiOfiWJ4++5YC3wV8HV\nSEuBg92HMrPFzE7pPq9lZotJ/H7dl3ypQXldA/4VeMPdf9jHsGi3V6avMMj1L6CWxLHEzcFX95U0\nU4F1oXGfJ3F1xDskDvFEXdfVJP6qaAUagSd71kXi6pstwdf2XKkrS9trAvAssCP4Pj6YXwX8SzD9\nceC1YHu9BtwSYT0nvX/guyT+YAEYCjwU/P/bAJwa9TZKsa6/D/4vbQGeB87MUF0PALuB9uD/1y3A\nbcBtwfMG3BvU/RpJrmTMYE13hLZVDPh4hrbVJ0gcptoa+r31+UxuL32yXURE0qJDWyIikhYFiYiI\npEVBIiIiaVGQiIhIWhQkIiKSFgWJSBJmdiTN5R8OPjWfbMwfLMmdk1Md02N8pZn9PtXxIulQkIhE\nJLjXUqm778z0a7t7M7DbzC7K9GtL8VGQiKQg+ETwD8xsmyX6vVwXzC8JboWx3cx+a2brzOzLwWI3\nEvpEvZn9Y3CDyO1m9t/7eJ0smk8VAAACNUlEQVQjZvZ/zOwVM3vWzCpDT19rZhvM7G0z+2QwfraZ\nvRiMf8XMPh4a/+ugBpFIKUhEUnMNsBA4F7gM+EFw+5BrgNnAOcDfABeGlrkI2BR6/G13rwIWABeb\n2YJeXmcE8Iq7LwL+CHwn9FyZuy8G/jY0vwm4PBh/HfCj0Pga4JMDf6siA1OUN20U+Qg+QeIuuJ1A\no5n9EbggmP+Qu3cBe8zs+dAyU4Dm0OOvBLf2Lwuem0/ithZhXcCvgulqIHwDvu7pTSTCC6Ac+LGZ\nLQQ6gXmh8U0kblUjEikFiUhq+moylaz51HES99DCzOYA3wAucPf9ZvaL7uf6Eb6HUWvwvZO//Oz+\nFxL3ODuXxBGGltD4oUENIpHSoS2R1LwAXGdmpcF5i0+RuLniSyQaP5WY2WQS7Va7vQGcHkyPBo4C\nB4Nxy/t4nRISd/8FuCFYfzJjgN3BHtFNJNrndptHbtz9WQqc9khEUvMYifMfW0jsJfxXd99jZo8A\nnyHxC/ttEp3pDgbL/I5EsDzj7lvM7FUSd4fdCfypj9c5CpxlZpuC9VzXT10/AR4xs2tJ3J33aOi5\nS4IaRCKlu/+KpMnMRnqiK94EEnspFwUhM4zEL/eLgnMrqazriLuPHKS6XgBWuPv+wVifSF+0RyKS\nvt+a2VigAvgf7r4HwN2Pm9l3SPTGrstkQcHhtx8qRCQTtEciIiJp0cl2ERFJi4JERETSoiAREZG0\nKEhERCQtChIREUmLgkRERNLy/wHQ0w5JpbsEJQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a1c2c1310>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('alpha is:', 1.0)\n"
     ]
    }
   ],
   "source": [
    "mse_mean = np.mean(ridge.cv_values_, axis = 0)\n",
    "plt.plot(np.log10(alphas), mse_mean.reshape(len(alphas),1)) \n",
    "\n",
    "#这是为了标出最佳参数的位置，不是必须\n",
    "#plt.plot(np.log10(ridge.alpha_)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "\n",
    "plt.xlabel('log(alpha)')\n",
    "plt.ylabel('mse')\n",
    "plt.show()\n",
    "\n",
    "print ('alpha is:', ridge.alpha_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>coef_lr</th>\n",
       "      <th>coef_ridge</th>\n",
       "      <th>columns</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.505543</td>\n",
       "      <td>0.452289</td>\n",
       "      <td>RAD_24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.297984</td>\n",
       "      <td>0.301090</td>\n",
       "      <td>RM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.184468</td>\n",
       "      <td>0.171709</td>\n",
       "      <td>RAD_7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.147403</td>\n",
       "      <td>0.143205</td>\n",
       "      <td>ZN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.147279</td>\n",
       "      <td>0.140265</td>\n",
       "      <td>RAD_8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.134516</td>\n",
       "      <td>0.140135</td>\n",
       "      <td>RAD_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.088869</td>\n",
       "      <td>0.088272</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.074143</td>\n",
       "      <td>0.075931</td>\n",
       "      <td>CHAS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.017002</td>\n",
       "      <td>0.010758</td>\n",
       "      <td>INDUS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>-0.001742</td>\n",
       "      <td>-0.004042</td>\n",
       "      <td>AGE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>-0.042599</td>\n",
       "      <td>-0.042286</td>\n",
       "      <td>RAD_4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>-0.056407</td>\n",
       "      <td>-0.053816</td>\n",
       "      <td>RAD_5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.104955</td>\n",
       "      <td>-0.101851</td>\n",
       "      <td>CRIM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.176803</td>\n",
       "      <td>-0.170743</td>\n",
       "      <td>NOX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-0.178692</td>\n",
       "      <td>-0.157054</td>\n",
       "      <td>TAX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>-0.200660</td>\n",
       "      <td>-0.177225</td>\n",
       "      <td>RAD_2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-0.209666</td>\n",
       "      <td>-0.205903</td>\n",
       "      <td>PTRATIO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>-0.272976</td>\n",
       "      <td>-0.264839</td>\n",
       "      <td>RAD_6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-0.361874</td>\n",
       "      <td>-0.357240</td>\n",
       "      <td>DIS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>-0.399165</td>\n",
       "      <td>-0.366233</td>\n",
       "      <td>RAD_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>-0.459577</td>\n",
       "      <td>-0.457144</td>\n",
       "      <td>LSTAT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     coef_lr  coef_ridge  columns\n",
       "20  0.505543    0.452289   RAD_24\n",
       "5   0.297984    0.301090       RM\n",
       "18  0.184468    0.171709    RAD_7\n",
       "1   0.147403    0.143205       ZN\n",
       "19  0.147279    0.140265    RAD_8\n",
       "14  0.134516    0.140135    RAD_3\n",
       "10  0.088869    0.088272        B\n",
       "3   0.074143    0.075931     CHAS\n",
       "2   0.017002    0.010758    INDUS\n",
       "6  -0.001742   -0.004042      AGE\n",
       "15 -0.042599   -0.042286    RAD_4\n",
       "16 -0.056407   -0.053816    RAD_5\n",
       "0  -0.104955   -0.101851     CRIM\n",
       "4  -0.176803   -0.170743      NOX\n",
       "8  -0.178692   -0.157054      TAX\n",
       "13 -0.200660   -0.177225    RAD_2\n",
       "9  -0.209666   -0.205903  PTRATIO\n",
       "17 -0.272976   -0.264839    RAD_6\n",
       "7  -0.361874   -0.357240      DIS\n",
       "12 -0.399165   -0.366233    RAD_1\n",
       "11 -0.459577   -0.457144    LSTAT"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 看看各特征的权重系数，系数的绝对值大小可视为该特征的重要性\n",
    "fs = pd.DataFrame({\"columns\":list(feat_names), \"coef_lr\":list((lr.coef_.T)), \"coef_ridge\":list((ridge.coef_.T))})\n",
    "fs.sort_values(by=['coef_lr'],ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3 正则化的线性回归（L1正则 --> Lasso）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The r2 score of LassoCV on test is 0.694867308919\n",
      "The r2 score of LassoCV on train is 0.754764862755\n"
     ]
    }
   ],
   "source": [
    "#### Lasso／L1正则\n",
    "# class sklearn.linear_model.LassoCV(eps=0.001, n_alphas=100, alphas=None, fit_intercept=True, \n",
    "#                                    normalize=False, precompute=’auto’, max_iter=1000, \n",
    "#                                    tol=0.0001, copy_X=True, cv=None, verbose=False, n_jobs=1,\n",
    "#                                    positive=False, random_state=None, selection=’cyclic’)\n",
    "from sklearn.linear_model import LassoCV\n",
    "\n",
    "#1. 设置超参数搜索范围\n",
    "#alphas = [ 0.01, 0.1, 1, 10,100]\n",
    "\n",
    "#2. 生成学习器实例\n",
    "#lasso = LassoCV(alphas=alphas)\n",
    "\n",
    "#1. 设置超参数搜索范围\n",
    "#Lasso可以自动确定最大的alpha，所以另一种设置alpha的方式是设置最小的alpha值（eps） 和 超参数的数目（n_alphas），\n",
    "#然后LassoCV对最小值和最大值之间在log域上均匀取值n_alphas个\n",
    "# np.logspace(np.log10(alpha_max * eps), np.log10(alpha_max),num=n_alphas)[::-1]\n",
    "\n",
    "#2 生成LassoCV实例（默认超参数搜索范围）\n",
    "lasso = LassoCV()  \n",
    "\n",
    "#3. 训练（内含CV）\n",
    "lasso.fit(X_train, y_train)  \n",
    "\n",
    "#4. 测试\n",
    "y_test_pred_lasso = lasso.predict(X_test)\n",
    "y_train_pred_lasso = lasso.predict(X_train)\n",
    "\n",
    "\n",
    "# 评估，使用r2_score评价模型在测试集和训练集上的性能\n",
    "print 'The r2 score of LassoCV on test is', r2_score(y_test, y_test_pred_lasso)\n",
    "print 'The r2 score of LassoCV on train is', r2_score(y_train, y_train_pred_lasso)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xt0XOV57/Hvo/vFsiRLsi1LNpaN\nMZhycTCGAD2BAsGQ1jQkaSBt2tA2tKshTdLmJM5JS1qyek4uvZ/SJCQll9M0QChJnMSEQBJKQoFY\ngMHYxliWjSXLtmTrfr/Mc/6YLTOIkT022rNnpN9nrVnae887e57tsfSb/b77Yu6OiIjIVDlRFyAi\nIplJASEiIkkpIEREJCkFhIiIJKWAEBGRpBQQIiKSlAJCRESSUkCIiEhSCggREUkqL+oCTlV1dbUv\nX7486jJERLLKM888c9Tda07lNVkXEMuXL6exsTHqMkREsoqZvXKqr1EXk4iIJKWAEBGRpBQQIiKS\nlAJCRESSUkCIiEhSCggREUlKASEiIkkpIEREMlws5vzvLbt4obU7re8bWkCY2T1m1m5mL07zvJnZ\nP5tZk5m9YGZvCqsWEZFstvtIH3c/3kxTe39a3zfMPYivARtO8Pz1wKrgcRvwhRBrERHJWk81HwPg\nkhVVaX3f0ALC3R8HOk/Q5EbgGx73FFBhZrVh1SMikq2eaj7GsgUl1FUUp/V9oxyDqANaEuZbg2Wv\nY2a3mVmjmTV2dHSkpTgRkUwQizlP7+vk0hUL0v7eUQaEJVnmyRq6+93uvs7d19XUnNLFCEVEstpL\nh/voHhzj0jR3L0G0AdEKLE2YrwfaIqpFRCQjRTX+ANEGxGbgd4OjmS4Fetz9UIT1iIhknKjGHyDE\n+0GY2beAK4FqM2sFPgXkA7j7F4EtwA1AEzAI3BpWLSIi2Why/GHDuYsjef/QAsLdbznJ8w58IKz3\nFxHJdi8d7qNnaIxLV6Z/gBp0JrWISMY6Pv7QkP7xB1BAiIhkrKeaj3FGVQlLIhh/AAWEiEhGOn7+\nQ0R7D6CAEBHJSE0d/fQMjbG+IZrxB1BAiIhkpJeP9AFwTu38yGpQQIiIZKC97QOYQUN1aWQ1KCBE\nRDLQ3o5+6iqKKS7IjawGBYSISAZqau9nZc28SGtQQIiIZJhYzGk+2s+ZCxUQIiKSoK1niOGxmPYg\nRETktfZ2DACwsia6AWpQQIiIZJzJe0+vVBeTiIgk2tvRT0VJPlWlBZHWoYAQEckwe4MjmMyS3Xgz\nfRQQIiIZZm9Hf+TjD6CAEBHJKN2DoxztH438CCZQQIiIZJTJI5iiPgcCFBAiIhll7+QRTLN9D8LM\nNpjZbjNrMrNNSZ4/w8x+YmYvmNljZlYfZj0iIplub0c/Bbk51FdGc5OgRKEFhJnlAncB1wNrgFvM\nbM2UZn8LfMPdzwfuBP5PWPWIiGSDvR39NFSXkpcbfQdPmBWsB5rcvdndR4F7gRuntFkD/CSY/lmS\n50VE5pSm9n5WLoz+CCYINyDqgJaE+dZgWaLngXcE028HyswsuvvriYhEaGR8ggOdgxkx/gDhBkSy\nMzx8yvxHgbeY2XPAW4CDwPjrVmR2m5k1mlljR0fHzFcqIpIBXjk2SMwzY4Aawg2IVmBpwnw90JbY\nwN3b3P0md18LfDJY1jN1Re5+t7uvc/d1NTU1IZYsIhKdpgw6ggnCDYitwCozazCzAuBmYHNiAzOr\nNrPJGj4B3BNiPSIiGW3PkX7MMuMcCAgxINx9HLgdeBjYBdzv7jvM7E4z2xg0uxLYbWYvA4uAvwmr\nHhGRTLenvY/6ymhvM5ooL8yVu/sWYMuUZXckTD8APBBmDSIi2aKpvZ9VC8uiLuO46A+0FRERxidi\nNB8dyJjuJVBAiIhkhJauIUbHYwoIERF5rT1H+gBYpYAQEZFEe4JDXLUHISIir7G3vZ/a8iLKivKj\nLuU4BYSISAbY096fUXsPoIAQEYlcLOY0KSBERGSqg91DDI1NZNQ5EKCAEBGJXFNHfIB61SLtQYiI\nSIKmI8ERTBlykb5JCggRkYjtae+jel4hlaUFUZfyGgoIEZGIxY9gyoy7yCVSQIiIRMjdM+4ifZMU\nECIiEWrvG6FveDzjBqhBASEiEqk9RzLvEhuTFBAiIhF66XAvAKsXqYtJREQS7GzrZdH8QqrmFUZd\nyusoIEREIrTzUC9raudHXUZSoQaEmW0ws91m1mRmm5I8v8zMfmZmz5nZC2Z2Q5j1iIhkkpHxCZra\n+1mzZI4FhJnlAncB1wNrgFvMbM2UZn8B3O/ua4GbgX8Nqx4RkUyz50g/4zFnTW151KUkFeYexHqg\nyd2b3X0UuBe4cUobByajsxxoC7EeEZGMsvNQfID6nNrMG6AGyAtx3XVAS8J8K3DJlDZ/BfzYzD4I\nlALXhFiPiEhG2dnWS0lBLmdUZd5Z1BDuHoQlWeZT5m8Bvubu9cANwP8zs9fVZGa3mVmjmTV2dHSE\nUKqISPrtOtTL2YvLyM1J9ucyemEGRCuwNGG+ntd3If0BcD+Auz8JFAHVU1fk7ne7+zp3X1dTUxNS\nuSIi6ePu8SOYMnSAGsINiK3AKjNrMLMC4oPQm6e0OQBcDWBm5xAPCO0iiMis19o1RN/wOOdk6CGu\nEGJAuPs4cDvwMLCL+NFKO8zsTjPbGDT7c+D9ZvY88C3gfe4+tRtKRGTWmRygztRzICDcQWrcfQuw\nZcqyOxKmdwKXh1mDiEgm2nWolxyDsxdnbkDoTGoRkQjsbOulobqU4oLcqEuZlgJCRCQCOw/1ZvT4\nAyggRETSrmdojNauoYw+ggkUECIiafdSFgxQgwJCRCTtXmwLAkJ7ECIikmhbSzdLyotYWFYUdSkn\npIAQEUmzbS1dXLisIuoyTkoBISKSRsf6R2jpHOLCpQoIERFJsK2lG4ALl1ZGXMnJKSBERNLouQPd\n5OYY59Vl5k2CEikgRETSaFtLN6sXlWX0GdSTFBAiImkSiznPt3RnxQA1KCBERNKm+Wg/fSPjWTFA\nDQoIEZG0ee5AfIB6rQJCREQSbWvppqwwj5U186IuJSUKCBGRNNnW0s35S8vJydB7UE+lgBARSYOh\n0QleOtyXNeMPoIAQEUmLF9t6mIh5VpwgNynUgDCzDWa228yazGxTkuf/wcy2BY+Xzaw7zHpERKLy\n3IEugKzagwjtntRmlgvcBVwLtAJbzWxzcB9qANz9IwntPwisDaseEZEo/XJfJw3VpdSUFUZdSsrC\n3INYDzS5e7O7jwL3AjeeoP0twLdCrEdEJBITMeeX+zq5pGFB1KWckjADog5oSZhvDZa9jpmdATQA\nPw2xHhGRSOw61Evv8DiXrqiKupRTEmZAJDuOy6dpezPwgLtPJF2R2W1m1mhmjR0dHTNWoIhIOjy9\nrxOAS1ZoD2JSK7A0Yb4eaJum7c2coHvJ3e9293Xuvq6mpmYGSxQRCd/TzcdYtqCE2vLiqEs5JWEG\nxFZglZk1mFkB8RDYPLWRma0GKoEnQ6xFRCQSsZjzy/2dXJplew8QYkC4+zhwO/AwsAu43913mNmd\nZrYxoektwL3uPl33k4hI1tp9pI/uwTEuaciu8QcI8TBXAHffAmyZsuyOKfN/FWYNIiJReqr5GJB9\n4w+gM6lFREL1dHMn9ZXF1FeWRF3KKUs5IMzsCjO7NZiuMbOG8MoSEcl+k+MP2di9BCkGhJl9Cvg4\n8IlgUT7w72EVJSIyG+xp76dzYDQrB6gh9T2ItwMbgQEAd28DysIqSkRkNnh6X3z8IdtOkJuUakCM\nBkcZOYCZlYZXkojI7PD4y0epqyimvjK7zn+YlGpA3G9mXwIqzOz9wKPAl8MrS0Qku42MT/Dfe49y\n5eoazLLjBkFTpXSYq7v/rZldC/QCq4E73P2RUCsTEclijfu7GByd4KrVC6Mu5bSlFBBBl9JP3f2R\n4Mzn1WaW7+5j4ZYnIpKdHtvdTkFuDpedmZ3jD5B6F9PjQKGZ1RHvXroV+FpYRYmIZLvHdnewvmEB\nJQWhno8cqlQDwtx9ELgJ+L/u/nZgTXhliYhkr4PdQ+xp7+fK1dl9cdGUA8LM3gz8NvDDYFn2xqKI\nSIge290OMGcC4kPAJuDB4IJ7urmPiMg0HtvdQV1FMStr5kVdyhuS6l7AIBADbjGz3yF+MyBdfVVE\nZIrR8Rj/3XSU31xbl7WHt05KNSC+CXwUeJF4UIiISBKN+zsZGJ3gyiw+vHVSqgHR4e7fD7USEZFZ\n4GeTh7euzN7DWyelGhCfMrOvAD8BRiYXuvuDoVQlIpKF3J2Hdxzh0pVVlBZm/3E8qW7BrcDZxK/i\nOtnF5IACQkQksKOtlwOdg3zgqpVRlzIjUg2IC9z9vFArERHJcg+9eIjcHOPaNYujLmVGpHqY61Nm\nphPjRESm4e5s2X6YN6+oYkFpQdTlzIhUA+IKYJuZ7TazF8xsu5m9cLIXmdmG4DVNZrZpmja/ZWY7\nzWyHmf3HqRQvIpIpXjrcx76jA1x/3uzYe4DUu5g2nOqKzSwXuAu4FmgFtprZZnffmdBmFfG71F3u\n7l1mlv3HhYnInPTQ9kPkGFx37hwLCHd/5TTWvR5ocvdmADO7F7gR2JnQ5v3AXe7eFbxP+2m8j4hI\n5La8eJhLGqqonlcYdSkzJtUuptNRB7QkzLcGyxKdBZxlZk+Y2VNmlnRPxcxuM7NGM2vs6OgIqVwR\nkdOz50gfTe393DCLupcg3IBIdo751Mtz5AGrgCuBW4CvmFnF617kfre7r3P3dTU12X3xKxGZfbZs\nP4zNsu4lCDcgWoGlCfP1QFuSNt9z9zF33wfsJh4YIiJZwd357raDXLx8AQvnF0VdzowKMyC2AqvM\nrMHMCoCbgc1T2nwXuArAzKqJdzk1h1iTiMiMeuaVLvYdHeCdF9VHXcqMCy0g3H0cuB14GNgF3B9c\nKvxOM9sYNHsYOGZmO4GfAf/T3Y+FVZOIyEz7dmMrJQW5vO282qhLmXGhXizE3bcAW6YsuyNh2oE/\nCx4iIlllcHScH7zQxg3n1c6Kay9NFWYXk4jIrPbQ9sMMjE7wrlnYvQQKCBGR0/btZ1o4o6qE9Q0L\noi4lFAoIEZHTcODYIE81d/LON9Vn/Z3jpqOAEBE5DQ8824oZvGOWdi+BAkJE5JSNTcS4f2sLV5xZ\nzZKK4qjLCY0CQkTkFD304mEO9w5z6+XLoy4lVAoIEZFTdM8v9tFQXcqVZ83uC1ArIERETsGzB7rY\n1tLN+y5bTk7O7BycnqSAEBE5Bff8Yh9lRXmz8tIaUykgRERS1NY9xEMvHubmi5fOyjOnp1JAiIik\n6BtPvoK787tvXh51KWmhgBARSUHP0BjffPoVrjt3MUsXlERdTlooIEREUvBvv9hH3/A4t//amVGX\nkjYKCBGRk+gZHOOrv9jHdecu4twl5VGXkzYKCBGRk/jKL5rpGxnnw9ecFXUpaaWAEBE5ge7BUb76\nxH5uOG8x59TOj7qctFJAiIicwJd/3szA6Dgfunpu7T2AAkJEZFpHeoeDvYdaVi8ui7qctAs1IMxs\ng5ntNrMmM9uU5Pn3mVmHmW0LHn8YZj0iIqfiMw+9xPiE87HrVkddSiRCOxXQzHKBu4BrgVZgq5lt\ndvedU5re5+63h1WHiMjpeOaVLr7z3EH+5MqVnFFVGnU5kQhzD2I90OTuze4+CtwL3Bji+4mIzIhY\nzLnz+ztYNL+QD1w1d857mCrMgKgDWhLmW4NlU73DzF4wswfMbGmI9YiIpOSBZ1t5vrWHTdefPSeu\nuTSdMAMi2XVwfcr894Hl7n4+8Cjw9aQrMrvNzBrNrLGjo2OGyxQReVX/yDif+9Fu3rSsgt+8MNl3\n2rkjzIBoBRL3COqBtsQG7n7M3UeC2S8DFyVbkbvf7e7r3H1dTU1NKMWKiAB88bG9HO0f4Y7fOBez\n2X2/h5MJMyC2AqvMrMHMCoCbgc2JDcysNmF2I7ArxHpERE7oUM8QX/55MxsvWMKFSyuiLidyoXWu\nufu4md0OPAzkAve4+w4zuxNodPfNwJ+a2UZgHOgE3hdWPSIiJ/P5h3fjwMc2zM3DWqcKdfTF3bcA\nW6YsuyNh+hPAJ8KsQUQkFS8e7OE7zx3kj/7HSuor58blvE9GZ1KLyJzn7vzND3dRWVLAn1y1Mupy\nMoYCQkTmvC3bD/Nk8zE+cs0q5hflR11OxlBAiMic1jc8xp0/2MG5S+bznkvOiLqcjDJ3zwAREQH+\n8dE9tPeN8KX3riM3Z24f1jqV9iBEZM7a2dbL1/57P+9Zv0yHtSahgBCROSkWc/7iu9upKM7nY9ed\nHXU5GUkBISJz0j1P7OPZA938rxvOobxEA9PJKCBEZM7Z2dbL5360m7euWcRNb5rb11s6EQWEiMwp\nw2MTfPi+5ygvyecz7zh/zl9v6UR0FJOIzCmf/dFLvHykn6/dejELSguiLiejaQ9CROaMH+84zFef\n2M/7LlvOlasXRl1OxlNAiMic8PKRPj5y3zbOry9n0/U6aikVCggRmfW6B0d5/zcaKS7I40vvvYii\n/NyoS8oKCggRmdXGJ2J88FvPcah7mC+99yJqy4ujLilraJBaRGatWMzZ9OB2fr7nKJ99x3lcdEZl\n1CVlFe1BiMis5O58+oc7eeCZVj509SreffGyqEvKOgoIEZmV/ukne/jqE/u59fLlfPiaVVGXk5XU\nxSQis4q78y8/beIfH93DOy+q5y/ftkYnw52mUPcgzGyDme02syYz23SCdu80MzezdWHWIyKzWyzm\n3PmDnfzdIy/z9rV1fOam88jRJbxPW2h7EGaWC9wFXAu0AlvNbLO775zSrgz4U+DpsGoRkdlvbCLG\nxx94gQefO8itly/nL9+2RuHwBoW5B7EeaHL3ZncfBe4FbkzS7tPA54DhEGsRkVnsWP8Iv/tvv+TB\n5w7y59eexR2/rnCYCWEGRB3QkjDfGiw7zszWAkvd/Qch1iEis9iLB3vY+C9P8MyBLv72XRfwwatX\nacxhhoQ5SJ3sE/LjT5rlAP8AvO+kKzK7DbgNYNkyHaomIvHB6Pu2tvCpzTuoKi3ggT9+M+fX665w\nMynMgGgFlibM1wNtCfNlwK8AjwVpvxjYbGYb3b0xcUXufjdwN8C6descEZnTOvpG+MSDL/DornYu\nP7OKf7p5LdXzCqMua9YJMyC2AqvMrAE4CNwMvGfySXfvAaon583sMeCjU8NBRGSSu/PD7Yf41Pd2\n0Dcyzl/++hpuvWy5xhtCElpAuPu4md0OPAzkAve4+w4zuxNodPfNYb23iMw+ezv6+avNO/j5nqOc\nV1fO3//WBaxaVBZ1WbNaqCfKufsWYMuUZXdM0/bKMGsRkezUPTjKFx7byz1P7KMoP5e/3nguv3Pp\nGeRqryF0OpNaRDLSwMg4X31iH196vJn+kXFuWlvPpuvPpqZMYw3pooAQkYzS3jvM15/czzefPkD3\n4BjXrlnER9+6mtWL1Z2UbgoIEYlcLOY82XyMbze28MPthxiPOdetWcwfvWUFa5fpEt1RUUCISCRi\nMWf7wR4e3nGY721r42D3EGVFedyyfhm/f3kDy6tLoy5xzlNAiEjadA+O8uTeY/y86Sg/3dXO4d5h\ncnOMK86s5uPXn81b1yzS7UAziAJCRELh7rxybJBtLd08e6CLZw90saOtF3coLcjlilXVfOzc1fza\n2QupKCmIulxJQgEhIjOib3iMxv1dPL2vkxdau3nxYA+9w+NAPBDOr6/gw1efxRWrqji/voL8XN2v\nLNMpIETklLk7bT3DPPtKfM/g2Ve62H6wh5hDfq5x9uL5vO38JZxXV87aZRWctahM5y1kIQWEiExr\nfCJGW/cw+48N8MqxAZqPDvDSoT52He6le3AMgKL8HM6vq+D2q87k0hVVrF1WSXGBxhFmAwWEyBw3\nuTfQ1N5Pc0c/zR0DQSAMcrB7iInYq9fHLM7PZfXiMq7/lVrOqS1j7dJKzq4tU3fRLKWAEJkDhkYn\nONw7zKGeIQ52DdHSOciBzkGajw7Q1N7P4OjE8bZlRXk0VJdyfn05v3FBLcsWlLC8qpTl1aUsLCvU\nvRbmEAWEyCzQPzJOU3s/+472s68j3hV0pHeYY/2jHO0fOT5YPCk3x1hSUcTyqlLeffFSzlw4jzNr\n5rGiZh7V8woUAgIoIESyxuR4wCudAzR3DMS7g4I9gEM9r96xN8dg6YISasuLWLNkPlWlBSycX0Rt\neRGLy4tYUl5MXWWxuoXkpBQQIhnC3Tk2MMr+owPsOzrAgc74GMCh7mHagq6h8YTxgNKCXFbUzOPS\nFVXxPYCF81hZU8qyBaUU5OmPv7xxCgiRNOsbHmP/0UH2dvSzd8qgcP/Iq11BuTnGorJCaiuKOa+u\nnLedV8sZVSUsW1DKihqNB0j4FBAiM2gi5hwbGOFIzwhHgkHh1u4h2rqHjw8Mdw6MHm+fm2PBIHAJ\nFy9fwLIFJTRUxweE69UNJBFTQIicwETM6RwY5djACEf7RukcHKVrYJTOgVG6Bl/9OTkY3DkwSmzK\nXdPzc40lFcXUVxZz3bmLOaMqHghnLpyn7iDJaAoImbPcna7BMQ52DXGwe4i24HGoJ97nf7hnmPa+\nkdecB5CooiSfBSUFVJTks3RBCWuXVVIzr4CaskIWzS9i0fz4oHDNvELdM1myUqgBYWYbgH8ifk/q\nr7j7Z6Y8/8fAB4AJoB+4zd13hlmTzB0j4xO098a7etp6ho8HQGtwHsDB7qHXHP8PUJiXw5KKYmrL\ni7hsZTW15UUsnF9IVWkhVfMKqCotYEFpAeXF+eSp+0dmudACwsxygbuAa4FWYKuZbZ4SAP/h7l8M\n2m8E/h7YEFZNMnsMj00Effyv/uE/2D3M4Z74HsDh3uHjl4JINL8oj/rKeD//FauqqQu6fuoqSlhS\nUcSCUp0DIDIpzD2I9UCTuzcDmNm9wI3A8YBw996E9qVA8n15mTNiMaejf4TWrkEOBwO97X0jtPcN\n09E3QkffCO19I68Z6J20oLSA2vIi6iuLueiMShYH3TyLyotYUl5EbUUx8wrVqyqSqjB/W+qAloT5\nVuCSqY3M7APAnwEFwK+FWI9kgMlv/m3d8SN8DvUM09o1SEvnEK1dg7R1DzM6EXvNa/JzjYVlRdSU\nFbJ0QQlvOqOS2qB/v7a8mCUVRSypKNaNZkRmWJgBkWw//XV7CO5+F3CXmb0H+Avg9163IrPbgNsA\nli1bNsNlyhs1EfPXHMnz6jf94Ft/sOxwz/DrLvkAUFVaQP2CEs6tK+e6X1lMfWUJ9RXF1FYUsais\niIqSfHX7iEQgzIBoBZYmzNcDbSdofy/whWRPuPvdwN0A69atUzfUDJmIOYOj4wyNTjA4OsFAMD0w\nOsHgyDgDoxMMjIzTHzz6hsfoHRqnd3iMnqExugfH6BocpWdoDE/yqRTm5VBTVsjCskIaqku5dEUV\ni+YXsbCskCUVxcEeQBElBer2EclEYf5mbgVWmVkDcBC4GXhPYgMzW+Xue4LZtwF7yGLuTszjf3hj\n7kzEnAl3YrFgOpifiDmxGIzHYsTcGY854xPx5eNBu/GJGGOTPyec8ViMsYkYY+PO6ER8enQ8eEzE\nGBmPMTI2wfBYjJHxCYaC6fjPyUeM4bH4cyNjsdd15ZxIfq5RWphHeXE+5cX5zC/Kp66imMqSAipL\n8qkue/VIn5qyQmrKCikrzNM3f5EsFlpAuPu4md0OPEz8MNd73H2Hmd0JNLr7ZuB2M7sGGAO6SNK9\nNFPu39rClx7fG+/j8nhf1+QfdMdxJ3g4DsQmn/PJdvH5mMfbTgaAO/EQCJZHIcegIC+HwrxcivLj\nP4vzcykqyKUoL4fKkgKK8nMoyg+W5+dSmJ9DSX4eJQXxdiX5uZQW5lJSEF9WWpjHvML49LyiPArz\n1L8vMteEum/v7luALVOW3ZEw/aEw3z9RZWkBZy+eDxYfHDEzchKmzcCI/8wxyJlcdrydkZvzarvc\nnMk28enc49Px9jk5Ru7x+fjP44+E+ZwcIz/4mZdj5OXmkBc8l59r5OXkkJtjFOblkJ+bQ16uUZCX\nQ0FufL4wL0fH44tIKOZM5++1axZx7ZpFUZchIpI19NVTRESSUkCIiEhSCggREUlKASEiIkkpIERE\nJCkFhIiIJKWAEBGRpBQQIiKSlHlU14c4TWbWAbyS5retBo6m+T3Dom3JPLNlO0DbkqmqgVJ3rzmV\nF2VdQETBzBrdfV3UdcwEbUvmmS3bAdqWTHW626IuJhERSUoBISIiSSkgUnN31AXMIG1L5pkt2wHa\nlkx1WtuiMQgREUlKexAiIpKUAiIJM/u0mb1gZtvM7MdmtmSadr9nZnuCR2h3w3sjzOzzZvZSsD3f\nMbOKadrtN7PtwTY3prvOVJzCtmwws91m1mRmm9Jd58mY2bvMbIeZxcxs2iNLsuQzSXVbMvozATCz\nBWb2SPD7/IiZVU7TbiL4TLaZ2eZ01zmdk/0bm1mhmd0XPP+0mS0/6UrdXY8pD2B+wvSfAl9M0mYB\n0Bz8rAymK6OuPUmdbwXygunPAp+dpt1+oDrqet/othC/ve1eYAVQADwPrIm69ik1ngOsBh4D1p2g\nXTZ8Jifdlmz4TII6PwdsCqY3neB3pT/qWk/n3xj4k8m/ZcDNwH0nW6/2IJJw996E2VLit7Ce6jrg\nEXfvdPcu4BFgQzrqOxXu/mN3Hw9mnwLqo6znjUhxW9YDTe7e7O6jwL3AjemqMRXuvsvdd0ddx0xI\ncVsy/jMJ3Ah8PZj+OvCbEdZyqlL5N07cvgeAq83MTrRSBcQ0zOxvzKwF+G3gjiRN6oCWhPnWYFkm\n+33goWmec+DHZvaMmd2WxppO13Tbko2fy3Sy7TOZTrZ8Jovc/RBA8HPhNO2KzKzRzJ4ys0wJkVT+\njY+3Cb5o9QBVJ1rpnLkn9VRm9iiwOMlTn3T377n7J4FPmtkngNuBT01dRZLXRnJI2Mm2JWjzSWAc\n+OY0q7nc3dvMbCHwiJm95O6Ph1Px9GZgWzLic0llO1KQNZ/JyVaRZFnG/a6cwmqWBZ/LCuCnZrbd\n3ffOTIWnLZV/41P+HOZsQLj7NSk2/Q/gh7w+IFqBKxPm64n3w6bdybYlGED/deBqDzogk6yjLfjZ\nbmbfIb7LmvY/RjOwLa3A0oTmCYIKAAAEFElEQVT5eqBt5ipMzSn8/zrROrLiM0lBRnwmcOJtMbMj\nZlbr7ofMrBZon2Ydk59Ls5k9Bqwl3v8fpVT+jSfbtJpZHlAOdJ5opepiSsLMViXMbgReStLsYeCt\nZlYZHO3w1mBZRjGzDcDHgY3uPjhNm1IzK5ucJr4tL6avytSksi3AVmCVmTWYWQHxwbiMOdIkVdny\nmaQoWz6TzcDk0Yi/B7xu7yj4fS8MpquBy4Gdaatweqn8Gydu3zuBn073hfG4qEffM/EB/CfxX8YX\ngO8DdcHydcBXEtr9PtAUPG6Nuu5ptqWJeL/jtuAxeRTDEmBLML2C+FEPzwM7iHcdRF776WxLMH8D\n8DLxb3UZty3A24l/mxsBjgAPZ/FnctJtyYbPJKixCvgJsCf4uSBYfvz3HrgM2B58LtuBP4i67hP9\nGwN3Ev9CBVAEfDv4PfolsOJk69SZ1CIikpS6mEREJCkFhIiIJKWAEBGRpBQQIiKSlAJCRESSUkDI\nnGFm/W/w9Q8EZ8+eqM1jJ7qqaaptprSvMbMfpdpeZKYoIERSYGbnArnu3pzu93b3DuCQmV2e7veW\nuU0BIXOOxX3ezF4M7rfw7mB5jpn9a3B/gx+Y2RYze2fwst8m4cxaM/tCcMG2HWb219O8T7+Z/Z2Z\nPWtmPzGzmoSn32VmvzSzl83sV4P2y83s50H7Z83ssoT23w1qEEkbBYTMRTcBFwIXANcAnw+uvXMT\nsBw4D/hD4M0Jr7kceCZh/pPuvg44H3iLmZ2f5H1KgWfd/U3Af/Ha63nluft64MMJy9uBa4P27wb+\nOaF9I/Crp76pIqdvzl6sT+a0K4BvufsEcMTM/gu4OFj+bXePAYfN7GcJr6kFOhLmfyu4BHde8Nwa\n4pdmSRQD7gum/x14MOG5yelniIcSQD7wL2Z2ITABnJXQvp345StE0kYBIXPRdDdJOdHNU4aIX8sG\nM2sAPgpc7O5dZva1yedOIvG6NiPBzwle/T38CPHrGV1AfO9+OKF9UVCDSNqoi0nmoseBd5tZbjAu\n8D+IX7zsF8A7grGIRbz2cu67gDOD6fnAANATtLt+mvfJIX7VTID3BOs/kXLgULAH817it5GcdBbZ\nezVXyVLag5C56DvExxeeJ/6t/mPuftjM/hO4mvgf4peBp4nfdQvi9wS5EnjU3Z83s+eIX2W1GXhi\nmvcZAM41s2eC9bz7JHX9K/CfZvYu4GfB6yddFdQgkja6mqtIAjOb5+79ZlZFfK/i8iA8ion/0b48\nGLtIZV397j5vhup6HLjR4/c/F0kL7UGIvNYPzKwCKAA+7e6HAdx9yMw+Rfy+vgfSWVDQDfb3CgdJ\nN+1BiIhIUhqkFhGRpBQQIiKSlAJCRESSUkCIiEhSCggREUlKASEiIkn9f96K1c7RgzdUAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a1c3a5550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('alpha is:', 0.00072929053930188251)\n"
     ]
    }
   ],
   "source": [
    "mses = np.mean(lasso.mse_path_, axis = 1)\n",
    "plt.plot(np.log10(lasso.alphas_), mses) \n",
    "#plt.plot(np.log10(lasso.alphas_)*np.ones(3), [0.3, 0.4, 1.0])\n",
    "plt.xlabel('log(alpha)')\n",
    "plt.ylabel('mse')\n",
    "plt.show()    \n",
    "            \n",
    "print ('alpha is:', lasso.alpha_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>coef_lasso</th>\n",
       "      <th>coef_lr</th>\n",
       "      <th>coef_ridge</th>\n",
       "      <th>columns</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.502616</td>\n",
       "      <td>0.505543</td>\n",
       "      <td>0.452289</td>\n",
       "      <td>RAD_24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.300666</td>\n",
       "      <td>0.297984</td>\n",
       "      <td>0.301090</td>\n",
       "      <td>RM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.202829</td>\n",
       "      <td>0.184468</td>\n",
       "      <td>0.171709</td>\n",
       "      <td>RAD_7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.141833</td>\n",
       "      <td>0.147403</td>\n",
       "      <td>0.143205</td>\n",
       "      <td>ZN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.172528</td>\n",
       "      <td>0.147279</td>\n",
       "      <td>0.140265</td>\n",
       "      <td>RAD_8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.174058</td>\n",
       "      <td>0.134516</td>\n",
       "      <td>0.140135</td>\n",
       "      <td>RAD_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.087695</td>\n",
       "      <td>0.088869</td>\n",
       "      <td>0.088272</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.075667</td>\n",
       "      <td>0.074143</td>\n",
       "      <td>0.075931</td>\n",
       "      <td>CHAS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.007140</td>\n",
       "      <td>0.017002</td>\n",
       "      <td>0.010758</td>\n",
       "      <td>INDUS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>-0.001587</td>\n",
       "      <td>-0.001742</td>\n",
       "      <td>-0.004042</td>\n",
       "      <td>AGE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.042599</td>\n",
       "      <td>-0.042286</td>\n",
       "      <td>RAD_4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>-0.009046</td>\n",
       "      <td>-0.056407</td>\n",
       "      <td>-0.053816</td>\n",
       "      <td>RAD_5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.101801</td>\n",
       "      <td>-0.104955</td>\n",
       "      <td>-0.101851</td>\n",
       "      <td>CRIM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.170409</td>\n",
       "      <td>-0.176803</td>\n",
       "      <td>-0.170743</td>\n",
       "      <td>NOX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-0.158660</td>\n",
       "      <td>-0.178692</td>\n",
       "      <td>-0.157054</td>\n",
       "      <td>TAX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>-0.128783</td>\n",
       "      <td>-0.200660</td>\n",
       "      <td>-0.177225</td>\n",
       "      <td>RAD_2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-0.205211</td>\n",
       "      <td>-0.209666</td>\n",
       "      <td>-0.205903</td>\n",
       "      <td>PTRATIO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>-0.221099</td>\n",
       "      <td>-0.272976</td>\n",
       "      <td>-0.264839</td>\n",
       "      <td>RAD_6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-0.355223</td>\n",
       "      <td>-0.361874</td>\n",
       "      <td>-0.357240</td>\n",
       "      <td>DIS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>-0.331377</td>\n",
       "      <td>-0.399165</td>\n",
       "      <td>-0.366233</td>\n",
       "      <td>RAD_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>-0.459030</td>\n",
       "      <td>-0.459577</td>\n",
       "      <td>-0.457144</td>\n",
       "      <td>LSTAT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    coef_lasso   coef_lr  coef_ridge  columns\n",
       "20    0.502616  0.505543    0.452289   RAD_24\n",
       "5     0.300666  0.297984    0.301090       RM\n",
       "18    0.202829  0.184468    0.171709    RAD_7\n",
       "1     0.141833  0.147403    0.143205       ZN\n",
       "19    0.172528  0.147279    0.140265    RAD_8\n",
       "14    0.174058  0.134516    0.140135    RAD_3\n",
       "10    0.087695  0.088869    0.088272        B\n",
       "3     0.075667  0.074143    0.075931     CHAS\n",
       "2     0.007140  0.017002    0.010758    INDUS\n",
       "6    -0.001587 -0.001742   -0.004042      AGE\n",
       "15    0.000000 -0.042599   -0.042286    RAD_4\n",
       "16   -0.009046 -0.056407   -0.053816    RAD_5\n",
       "0    -0.101801 -0.104955   -0.101851     CRIM\n",
       "4    -0.170409 -0.176803   -0.170743      NOX\n",
       "8    -0.158660 -0.178692   -0.157054      TAX\n",
       "13   -0.128783 -0.200660   -0.177225    RAD_2\n",
       "9    -0.205211 -0.209666   -0.205903  PTRATIO\n",
       "17   -0.221099 -0.272976   -0.264839    RAD_6\n",
       "7    -0.355223 -0.361874   -0.357240      DIS\n",
       "12   -0.331377 -0.399165   -0.366233    RAD_1\n",
       "11   -0.459030 -0.459577   -0.457144    LSTAT"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 看看各特征的权重系数，系数的绝对值大小可视为该特征的重要性\n",
    "fs = pd.DataFrame({\"columns\":list(feat_names), \"coef_lr\":list((lr.coef_.T)), \"coef_ridge\":list((ridge.coef_.T)), \"coef_lasso\":list((lasso.coef_.T))})\n",
    "fs.sort_values(by=['coef_lr'],ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
